pi-ui-extend 0.1.60 → 0.1.62

package/external/pi-tools-suite/README.md

@@ -362,6 +362,22 @@ npm run test:async-subagents-selection-e2e
 npm run test:e2e
 ```
 
+### Prompt evaluations
+
+Prompt evaluations are opt-in because they call a real model. They cover model-facing behavior that deterministic tests cannot prove: tool selection for `todo` and `compress`, async-subagent delegation/lifecycle boundaries, default internal role routing, ultrawork classification, and DCP summary retention. They are intentionally excluded from `npm test`.
+
+```bash
+# Full prompt-eval suite
+npm run test:prompt-evals
+
+# Focused suites
+npm run test:prompt-evals:tool-selection
+npm run test:prompt-evals:async
+npm run test:prompt-evals:dcp
+```
+
+The default live model is `zai/glm-5-turbo`. Override it for the whole suite with `PI_TOOLS_SUITE_E2E_MODEL=provider/model`, or use the existing component variables such as `TOOL_SELECTION_E2E_MODEL`, `ASYNC_SUBAGENTS_MODEL`, `ASYNC_SUBAGENTS_ROUTING_E2E_MODEL`, and `DCP_SUMMARY_E2E_MODEL`. The normal deterministic coverage remains `npm test`; run prompt evals after changing tool descriptions, routing/classifier prompts, DCP summary prompts, or the default evaluation model.
+
 Supporting docs and historical standalone README content are kept in `docs/`; third-party license texts are kept in `licenses/`.
 
 ## SDK pin

package/external/pi-tools-suite/package.json

@@ -24,6 +24,10 @@
     "test": "bun test test",
     "test:async-subagents-e2e": "ASYNC_SUBAGENTS_E2E=1 ASYNC_SUBAGENTS_DEBUG_LOGS=1 ASYNC_SUBAGENTS_MODEL=zai/glm-5-turbo bun test --concurrent --max-concurrency=30 test/async-subagents",
     "test:async-subagents-selection-e2e": "ASYNC_SUBAGENTS_SELECTION_E2E=1 ASYNC_SUBAGENTS_MODEL=zai/glm-5-turbo bun test --concurrent --max-concurrency=30 test/async-subagents/selection-e2e.test.ts",
+    "test:prompt-evals:tool-selection": "PROMPT_EVAL_E2E=1 bun test --concurrent --max-concurrency=10 test/tool-selection-e2e.test.ts",
+    "test:prompt-evals:async": "PROMPT_EVAL_E2E=1 bun test --concurrent --max-concurrency=5 test/async-subagents/selection-e2e.test.ts test/prompt-evals/async-routing-e2e.test.ts",
+    "test:prompt-evals:dcp": "PROMPT_EVAL_E2E=1 bun test --concurrent --max-concurrency=5 test/prompt-evals/dcp-summary-e2e.test.ts",
+    "test:prompt-evals": "PROMPT_EVAL_E2E=1 bun test --concurrent --max-concurrency=5 test/tool-selection-e2e.test.ts test/async-subagents/selection-e2e.test.ts test/prompt-evals",
     "bench:locate": "PI_LOCATE_BENCH_ITERATIONS=5 PI_LOCATE_BENCH_FAKE_IDX=0 PI_LOCATE_BENCH_MODEL=zai/glm-5-turbo PI_LOCATE_BENCH_MODES=direct-read-grep,ast-structural,repo-search-hybrid,repo-discovery,subagent-search,unrestricted-suite node test/fixtures/hard-to-find-project/benchmark/run-locate-benchmark.mjs",
     "bench:locate:analyze": "node test/fixtures/hard-to-find-project/benchmark/analyze-locate-benchmark.mjs",
     "test:locate-benchmark-e2e": "PI_LOCATE_BENCH_E2E=1 PI_LOCATE_BENCH_MODEL=zai/glm-5-turbo bun test test/locate-benchmark-e2e.test.ts",

package/external/pi-tools-suite/src/dcp/auto-compress.ts

@@ -116,7 +116,7 @@ export function buildProgrammaticSummary(
 	return lines.join("\n")
 }
 
-const SUMMARIZER_SYSTEM_PROMPT = `You summarize a slice of a coding agent's conversation so it can replace the raw messages in context. Produce a dense, continuation-focused summary: preserve user intent, decisions made, files/symbols changed or inspected, exact errors still actionable, verification status, and next steps. Drop full logs, repeated output, and incidental detail. Be concise (roughly 4-10 bullets). Output ONLY the summary text, no preamble.`
+const SUMMARIZER_SYSTEM_PROMPT = `You summarize a slice of a coding agent's conversation so it can replace the raw messages in context. Produce a dense, continuation-focused summary: preserve user intent, decisions made, files/symbols changed or inspected, exact errors still actionable, verification status, and next steps. Do not infer, invent, or add facts absent from the source; preserve uncertainty instead of filling gaps. Drop full logs, repeated output, and incidental detail. Be concise (roughly 4-10 bullets). Output ONLY the summary text, no preamble.`
 
 /** Outcome of one summarizer-model attempt, surfaced in DCP debug logs. */
 export interface ModelSummaryAttempt {

package/external/pi-tools-suite/src/dcp/prompts.ts

@@ -25,7 +25,7 @@ Do not compress active work, still-needed raw context, or material whose exact c
 
 DCP reminders: handle critical/high-context reminders promptly and compress any safe high-yield closed slice before more exploration; routine reminders mean compress only if a safe, closed, useful slice exists, otherwise continue the next atomic step and re-check later.
 
-Summaries must preserve only what is needed to continue: user intent and constraints, accepted decisions, files/symbols changed or inspected, actionable errors, verification status, and next steps. Drop incidental transcript detail, duplicate outputs, full logs, long code/JSON/diffs, and prose not needed later; include short literals only when required.
+Summaries must preserve only what is needed to continue: user intent and constraints, accepted decisions, files/symbols changed or inspected, actionable errors, verification status, and next steps. Do not infer, invent, or add facts that are not present in the selected messages. Drop incidental transcript detail, duplicate outputs, full logs, long code/JSON/diffs, and prose not needed later; include short literals only when required.
 `.trim()
 
 /**
@@ -55,6 +55,8 @@ If a \`<dcp-system-reminder>\` is present in context, treat it as a signal to ev
 THE SUMMARY
 Your summary must be COMPLETE FOR CONTINUATION, not a transcript rewrite. Preserve only information that will plausibly matter later: user intent, accepted constraints, decisions, files/symbols changed or inspected, exact errors that are still actionable, verification status, and next steps.
 
+Do not infer, invent, or add facts that are not present in the selected range. If the source is uncertain or incomplete, preserve that uncertainty instead of filling gaps.
+
 If active unfinished work exists, start with \`Active objective\` and \`Next step\`.
 
 Default to a compact structured summary (roughly 4-10 bullets for a normal completed work slice). Grow beyond that only when the compressed range contains multiple independent decisions, unresolved blockers, or precise state that is genuinely required to continue.

package/external/pi-tools-suite/src/tool-descriptions.ts

@@ -220,6 +220,7 @@ export const TODO_TOOL_DESCRIPTION: ToolDescription = {
 	promptGuidelines: [
 		"Use `todo` for complex work with 3+ steps, explicit user task lists, or new non-trivial requirements. Skip single trivial tasks and purely conversational requests.",
 		"For multi-step implementation/debugging plans, include a final user-facing report todo in the initial plan with acceptance criteria for changed files/behavior, verification results, and remaining manual actions; close it immediately before the final response, never via compression.",
+		"When create or batch_create already sets the intended status, do not issue a redundant update with the same status; continue the work instead.",
 		"Resync before continuing when user/new findings change scope, requirements, safety, feasibility, approach, dependencies, or order; update tasks/blockers and defer obsolete work.",
 		"Update todos when starting, finishing, blocking, splitting, abandoning, or materially changing a step; before planned work mark exactly one in_progress with activeForm and complete it only after verification.",
 		"If partial, tests fail, or blocked, keep the task in_progress and add/update a blocker. Never use `clear`, `delete`, or batch deletion to hide unfinished/stale/forgotten todos; delete only on explicit request or creation mistake.",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-ui-extend",
-  "version": "0.1.60",
+  "version": "0.1.62",
   "private": false,
   "type": "module",
   "bin": {