pi-skill-evolve 0.1.8 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/skill-evolve/fixtures/selftest/good-description/after.md +9 -0
- package/extensions/skill-evolve/fixtures/selftest/good-description/change.diff +13 -0
- package/extensions/skill-evolve/fixtures/selftest/good-description/fixture.json +6 -0
- package/extensions/skill-evolve/lib/replay-fixture.ts +5 -2
- package/extensions/skill-evolve/lib/review-post.ts +92 -21
- package/extensions/skill-evolve/lib/review-prompt.ts +5 -1
- package/extensions/skill-evolve/lib/review-record.ts +14 -1
- package/extensions/skill-evolve/lib/review-supervisor.ts +13 -1
- package/extensions/skill-evolve/lib/runs.ts +4 -0
- package/extensions/skill-evolve/lib/selftest.ts +22 -5
- package/extensions/skill-evolve/lib/web/applied.ts +6 -1
- package/extensions/skill-evolve/lib/web/approvals.ts +7 -0
- package/extensions/skill-evolve/lib/web/done.ts +20 -6
- package/extensions/skill-evolve/lib/web/filter.ts +0 -15
- package/extensions/skill-evolve/lib/web/pending.ts +7 -1
- package/extensions/skill-evolve/lib/web/render-controls.ts +7 -16
- package/extensions/skill-evolve/lib/web/render-rows.ts +11 -2
- package/extensions/skill-evolve/lib/web/render-run.ts +46 -10
- package/extensions/skill-evolve/lib/web/reports.ts +11 -3
- package/extensions/skill-evolve/lib/web/run-detail.ts +41 -13
- package/extensions/skill-evolve/lib/web/style.ts +2 -0
- package/extensions/skill-evolve/test/review-post.test.ts +53 -6
- package/extensions/skill-evolve/test/review-prompt.test.ts +9 -0
- package/extensions/skill-evolve/test/review-record.test.ts +59 -0
- package/extensions/skill-evolve/test/review-supervisor.test.ts +15 -1
- package/extensions/skill-evolve/test/selftest.test.ts +51 -16
- package/extensions/skill-evolve/test/web-filter-labels.test.ts +31 -28
- package/extensions/skill-evolve/test/web-redesign.test.ts +1 -1
- package/extensions/skill-evolve/test/web-render-run.test.ts +70 -1
- package/extensions/skill-evolve/test/web-run-detail.test.ts +27 -0
- package/extensions/skill-evolve/test/web-view-coverage.test.ts +100 -0
- package/package.json +1 -1
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import type { WebState } from "./data.ts";
|
|
10
10
|
import { DIFF_PREVIEW_LINES, renderDiffLines, parseDiffLines } from "./diff-lines.ts";
|
|
11
|
-
import type { RunBlocking, RunDetail } from "./run-detail.ts";
|
|
11
|
+
import type { RunBlocking, RunDetail, RunPassing } from "./run-detail.ts";
|
|
12
12
|
import { sessionHref } from "./run-session.ts";
|
|
13
13
|
import { shellPage } from "./shell.ts";
|
|
14
14
|
import { escapeHtml, formatStamp } from "./text.ts";
|
|
@@ -89,20 +89,42 @@ function candidateSection(detail: RunDetail): string {
|
|
|
89
89
|
</section>`;
|
|
90
90
|
}
|
|
91
91
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
92
|
+
/** 旧 run 的兼容文案:03-review.json 没有 summary / passed 时明说「不是页面坏了」 */
|
|
93
|
+
const LEGACY_REVIEW_NOTE = "该 run 由旧版评审产出,没有结论摘要(重跑一次就有)";
|
|
94
|
+
/** 新版评审也确实可能两块全空(比如 skipped):那时不能说「旧版」,要说「没给理由」 */
|
|
95
|
+
const NO_REASON_NOTE = "03-review.json 里没有 blocking / passed:这一轮评审只写了结论,没写理由。";
|
|
96
|
+
|
|
97
|
+
/** 理由表的一行:blocking 用 issue、passed 用 point,但三列形状一样(说明 / 证据位置 / 原文引用) */
|
|
98
|
+
interface ReasonRow {
|
|
99
|
+
text: string;
|
|
100
|
+
file: string;
|
|
101
|
+
line: number | null;
|
|
102
|
+
quote: string;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function reasonTable(head: string, rows: readonly ReasonRow[]): string {
|
|
106
|
+
if (rows.length === 0) return "";
|
|
107
|
+
const body = rows.map((row) => {
|
|
108
|
+
const where = row.file.length > 0 ? `${row.file}${row.line === null ? "" : `:${row.line}`}` : "(未写位置)";
|
|
109
|
+
return `<tr><td>${escapeHtml(row.text)}</td><td><code>${escapeHtml(where)}</code></td><td>${escapeHtml(row.quote)}</td></tr>`;
|
|
97
110
|
});
|
|
98
|
-
return `<
|
|
99
|
-
<
|
|
111
|
+
return `<h3>${head}</h3>
|
|
112
|
+
<table class="tbl">
|
|
113
|
+
<thead><tr><th>${head === "阻断意见" ? "问题" : "通过点"}</th><th>证据位置</th><th>原文引用</th></tr></thead>
|
|
100
114
|
<tbody>
|
|
101
|
-
${
|
|
115
|
+
${body.join("\n")}
|
|
102
116
|
</tbody>
|
|
103
117
|
</table>`;
|
|
104
118
|
}
|
|
105
119
|
|
|
120
|
+
function blockingRows(blocking: readonly RunBlocking[]): ReasonRow[] {
|
|
121
|
+
return blocking.map((item) => ({ text: item.issue, file: item.file, line: item.line, quote: item.quote }));
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function passedRows(passed: readonly RunPassing[]): ReasonRow[] {
|
|
125
|
+
return passed.map((item) => ({ text: item.point, file: item.file, line: item.line, quote: item.quote }));
|
|
126
|
+
}
|
|
127
|
+
|
|
106
128
|
function reviewSection(detail: RunDetail): string {
|
|
107
129
|
if (detail.review === null) {
|
|
108
130
|
return `<section>
|
|
@@ -111,6 +133,17 @@ function reviewSection(detail: RunDetail): string {
|
|
|
111
133
|
</section>`;
|
|
112
134
|
}
|
|
113
135
|
const review = detail.review;
|
|
136
|
+
// 结论摘要:用户问的第一句永远是「为什么」,所以它排在 verdict 那一行之后、理由表之前,且字号大一档
|
|
137
|
+
// legacy 看的是 03-review.json 里有没有新契约的键,不是「summary 空不空」:
|
|
138
|
+
// 新版评审失败时同样没有 summary,那时要说「没给理由」,不能说成旧版
|
|
139
|
+
const legacy = review.legacy;
|
|
140
|
+
const summary = legacy
|
|
141
|
+
? `<p class="hint">${LEGACY_REVIEW_NOTE}</p>`
|
|
142
|
+
: `<p class="verdict-summary">${escapeHtml(review.summary ?? "")}</p>`;
|
|
143
|
+
const blocked = reasonTable("阻断意见", blockingRows(detail.blocking));
|
|
144
|
+
const passed = reasonTable("通过理由", passedRows(detail.passed));
|
|
145
|
+
// 旧 run 已经有上面那句兼容文案了,不再叠一句「没给理由」
|
|
146
|
+
const emptyNote = !legacy && blocked === "" && passed === "" ? `<p class="hint">${NO_REASON_NOTE}</p>` : "";
|
|
114
147
|
return `<section>
|
|
115
148
|
<h2>评审结论</h2>
|
|
116
149
|
<p class="kv-line">
|
|
@@ -119,7 +152,10 @@ action <b>${escapeHtml(review.action)}</b> ·
|
|
|
119
152
|
状态 ${escapeHtml(review.status)} ·
|
|
120
153
|
评审模型 <code>${escapeHtml(review.reviewerModel)}</code>
|
|
121
154
|
</p>
|
|
122
|
-
${
|
|
155
|
+
${summary}
|
|
156
|
+
${emptyNote}
|
|
157
|
+
${blocked}
|
|
158
|
+
${passed}
|
|
123
159
|
</section>`;
|
|
124
160
|
}
|
|
125
161
|
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
import * as fs from "node:fs";
|
|
6
6
|
import * as path from "node:path";
|
|
7
7
|
import { readVerify } from "../apply.ts";
|
|
8
|
-
import type { RunSummary } from "../runs.ts";
|
|
8
|
+
import type { RunStatus, RunSummary } from "../runs.ts";
|
|
9
9
|
import { readRunError } from "../stale.ts";
|
|
10
10
|
import { asNumber, asRecord, asText, readJson, reasonLabel } from "./json-read.ts";
|
|
11
11
|
import { sessionFromRecord, type SessionRef } from "./run-session.ts";
|
|
@@ -13,6 +13,12 @@ import { sessionFromRecord, type SessionRef } from "./run-session.ts";
|
|
|
13
13
|
/** 单条报告里最多列几个技能:全量列会把 details 撑成第二份 findings */
|
|
14
14
|
export const MAX_FINDING_ROWS = 5;
|
|
15
15
|
|
|
16
|
+
/**
|
|
17
|
+
* 复盘报告页唯一由 run 状态决定的分支:status=stale 且没有 findings 的失败 run 也要占一行。
|
|
18
|
+
* (其余 run 靠「能读到 01-findings.json」入场,与状态无关)导出给覆盖性测试用。
|
|
19
|
+
*/
|
|
20
|
+
export const REPORTS_RUN_STATUSES: readonly RunStatus[] = ["stale"];
|
|
21
|
+
|
|
16
22
|
export interface FindingRow {
|
|
17
23
|
skill: string;
|
|
18
24
|
kind: string;
|
|
@@ -126,13 +132,15 @@ export function buildReports(evolutionDir: string, runs: readonly RunSummary[]):
|
|
|
126
132
|
// 失败也要有一行档案:每条 run 都能在报告里找到,不能因为没产出就从历史里消失
|
|
127
133
|
const summary = summaries.get(runId);
|
|
128
134
|
const raw = readJson<Record<string, unknown>>(path.join(runDir, "run.json"));
|
|
129
|
-
|
|
135
|
+
const status = summary?.status ?? asText(raw?.status);
|
|
136
|
+
// 判据用清单本体而不是写死 "stale":清单改了代码就跟着改(同类写法见 ./applied.ts)
|
|
137
|
+
if (!REPORTS_RUN_STATUSES.some((known) => known === status)) continue;
|
|
130
138
|
const error = readRunError(runDir);
|
|
131
139
|
const tail = error?.childLogTail ?? [];
|
|
132
140
|
const lastLine = tail.length > 0 ? tail[tail.length - 1]!.trim() : "";
|
|
133
141
|
reports.push({
|
|
134
142
|
runId,
|
|
135
|
-
status
|
|
143
|
+
status,
|
|
136
144
|
trigger: meta.trigger,
|
|
137
145
|
session: meta.session,
|
|
138
146
|
startedAt: summary?.dispatchedAt ?? asNumber(raw?.dispatchedAt),
|
|
@@ -33,6 +33,14 @@ export interface RunBlocking {
|
|
|
33
33
|
quote: string;
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
+
/** 通过理由(03-review.json 的 passed):哪一点没问题 + 支撑它的原文 */
|
|
37
|
+
export interface RunPassing {
|
|
38
|
+
point: string;
|
|
39
|
+
file: string;
|
|
40
|
+
line: number | null;
|
|
41
|
+
quote: string;
|
|
42
|
+
}
|
|
43
|
+
|
|
36
44
|
export interface RunGate {
|
|
37
45
|
gate: string;
|
|
38
46
|
name: string;
|
|
@@ -70,8 +78,18 @@ export interface RunDetail {
|
|
|
70
78
|
/** candidate 目录里的技能名(与 skill 一致才读全文) */
|
|
71
79
|
candidateSkill: string | null;
|
|
72
80
|
candidateText: string | null;
|
|
73
|
-
review: {
|
|
81
|
+
review: {
|
|
82
|
+
status: string;
|
|
83
|
+
verdict: string;
|
|
84
|
+
action: string;
|
|
85
|
+
reviewerModel: string;
|
|
86
|
+
summary: string | null;
|
|
87
|
+
/** 03-review.json 里连 summary / passed 两个键都没有 = v0.1.9 之前写的记录(详情页要说旧版,而不是说「没给理由」) */
|
|
88
|
+
legacy: boolean;
|
|
89
|
+
} | null;
|
|
74
90
|
blocking: RunBlocking[];
|
|
91
|
+
/** 通过理由:旧 run 没有这个字段,读出来是空数组 */
|
|
92
|
+
passed: RunPassing[];
|
|
75
93
|
gates: RunGate[];
|
|
76
94
|
verifyStatus: string | null;
|
|
77
95
|
findings: RunFinding[];
|
|
@@ -95,30 +113,40 @@ function exists(file: string): boolean {
|
|
|
95
113
|
}
|
|
96
114
|
}
|
|
97
115
|
|
|
98
|
-
/** 03-review.json →
|
|
99
|
-
function
|
|
100
|
-
const
|
|
101
|
-
|
|
102
|
-
const blocking: RunBlocking[] = [];
|
|
103
|
-
for (const item of Array.isArray(raw.blocking) ? raw.blocking : []) {
|
|
116
|
+
/** 03-review.json 里的证据项 → 行上的 file / line / quote(blocking 与 passed 共用同一形状) */
|
|
117
|
+
function readEvidenceRows<T>(items: unknown, build: (text: string, file: string, line: number | null, quote: string) => T): T[] {
|
|
118
|
+
const rows: T[] = [];
|
|
119
|
+
for (const item of Array.isArray(items) ? items : []) {
|
|
104
120
|
const record = asRecord(item);
|
|
105
121
|
if (!record) continue;
|
|
106
122
|
const evidence = asRecord(record.evidence);
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
file: asText(evidence?.file),
|
|
110
|
-
line: asNumber(evidence?.line),
|
|
111
|
-
quote: asText(evidence?.quote),
|
|
112
|
-
});
|
|
123
|
+
const text = asText(record.issue) || asText(record.point) || "(未写说明)";
|
|
124
|
+
rows.push(build(text, asText(evidence?.file), asNumber(evidence?.line), asText(evidence?.quote)));
|
|
113
125
|
}
|
|
126
|
+
return rows;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* 03-review.json → verdict / action / reviewerModel / summary + blocking + passed。
|
|
131
|
+
* 旧版评审产出的文件没有 summary / passed:summary 读成 null、passed 读成空数组(不抛、不补造)。
|
|
132
|
+
*/
|
|
133
|
+
function readReview(runDir: string): Pick<RunDetail, "review" | "blocking" | "passed"> {
|
|
134
|
+
const raw = asRecord(readJson<unknown>(path.join(runDir, "03-review.json")));
|
|
135
|
+
if (!raw) return { review: null, blocking: [], passed: [] };
|
|
136
|
+
const blocking = readEvidenceRows<RunBlocking>(raw.blocking, (issue, file, line, quote) => ({ issue, file, line, quote }));
|
|
137
|
+
const passed = readEvidenceRows<RunPassing>(raw.passed, (point, file, line, quote) => ({ point, file, line, quote }));
|
|
114
138
|
return {
|
|
115
139
|
review: {
|
|
116
140
|
status: asText(raw.status, "(未写状态)"),
|
|
117
141
|
verdict: asText(raw.verdict, "(未写 verdict)"),
|
|
118
142
|
action: asText(raw.action, "(未写 action)"),
|
|
119
143
|
reviewerModel: asText(raw.reviewerModel, "(未写模型)"),
|
|
144
|
+
summary: asText(raw.summary, "") || null,
|
|
145
|
+
// 两个键都不存在才算旧记录:新版即使评审失败也会写 passed: [](空数组也是一个键),不能混谈
|
|
146
|
+
legacy: raw.summary === undefined && raw.passed === undefined,
|
|
120
147
|
},
|
|
121
148
|
blocking,
|
|
149
|
+
passed,
|
|
122
150
|
};
|
|
123
151
|
}
|
|
124
152
|
|
|
@@ -113,6 +113,8 @@ a.btn { display: inline-block; color: var(--blue); padding: 1px 8px; border: 1px
|
|
|
113
113
|
.kv .k { color: var(--muted); font-size: 12px }
|
|
114
114
|
.kv .v { color: var(--fg) }
|
|
115
115
|
.kv-line { margin: 4px 0 }
|
|
116
|
+
/* 评审结论摘要:详情页里唯一「大一号」的正文,用户上来先读这一句 */
|
|
117
|
+
.verdict-summary { margin: 6px 0; font-size: 15px; font-weight: 600; color: var(--fg) }
|
|
116
118
|
a.run { color: #c8d3e6; border-bottom: 1px dashed #45505f }
|
|
117
119
|
a.run:hover { border-bottom-color: var(--blue); text-decoration: none }
|
|
118
120
|
a.sess { color: var(--muted); border-bottom: 1px dashed #45505f }
|
|
@@ -1,21 +1,27 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* 评审后处理单测(设计稿 12.3):结构、证据、一致性一律代码判,不信 LLM 自觉。
|
|
3
3
|
* 每条用例对应「模型输出的一种坏法」,覆盖判 invalid / reject 的每一道关。
|
|
4
|
+
* 「结论必须写为什么」也在这里钉住:没有 summary、pass 却没有 passed(或 reject 却没有 blocking)一律 invalid。
|
|
4
5
|
*/
|
|
5
6
|
import { test } from "node:test";
|
|
6
7
|
import assert from "node:assert/strict";
|
|
7
|
-
import { parseReviewJson, postProcessReview } from "../lib/review-post.ts";
|
|
8
|
+
import { parseReviewJson, postProcessReview, SUMMARY_MAX } from "../lib/review-post.ts";
|
|
8
9
|
|
|
9
|
-
const
|
|
10
|
-
const withBlocking = (verdict: string, evidence: unknown): string =>
|
|
11
|
-
JSON.stringify({ verdict, blocking: [{ issue: "有问题", evidence }], notes: "" });
|
|
10
|
+
const SUMMARY = "新描述补了触发场景词,没删原有禁令";
|
|
12
11
|
const EVIDENCE = { file: ".pi/skills/demo/SKILL.md", line: 12, quote: "允许用 find" };
|
|
12
|
+
const PASSED = [{ point: "原有禁止项仍完整保留", evidence: EVIDENCE }];
|
|
13
|
+
const GOOD_PASS = JSON.stringify({ verdict: "pass", summary: SUMMARY, blocking: [], passed: PASSED, notes: "没问题" });
|
|
14
|
+
const withBlocking = (verdict: string, evidence: unknown): string =>
|
|
15
|
+
JSON.stringify({ verdict, summary: SUMMARY, blocking: [{ issue: "有问题", evidence }], passed: [], notes: "" });
|
|
13
16
|
|
|
14
|
-
test("合法 pass 返回 ok", () => {
|
|
17
|
+
test("合法 pass 返回 ok,且 summary / passed 原样带出", () => {
|
|
15
18
|
const result = postProcessReview(GOOD_PASS);
|
|
16
19
|
assert.equal(result.status, "ok");
|
|
17
20
|
assert.equal(result.verdict, "pass");
|
|
18
21
|
assert.deepEqual(result.blocking, []);
|
|
22
|
+
assert.equal(result.summary, SUMMARY);
|
|
23
|
+
assert.equal(result.passed.length, 1);
|
|
24
|
+
assert.equal(result.passed[0].point, "原有禁止项仍完整保留");
|
|
19
25
|
});
|
|
20
26
|
|
|
21
27
|
test("带 ```json 围栏与前后废话也能抠出 JSON", () => {
|
|
@@ -71,11 +77,52 @@ test("pass 却带 blocking 判 invalid:contradiction", () => {
|
|
|
71
77
|
});
|
|
72
78
|
|
|
73
79
|
test("reject 带完整证据返回 ok 且保留 blocking/notes", () => {
|
|
74
|
-
const
|
|
80
|
+
const text = JSON.stringify({ verdict: "reject", summary: SUMMARY, blocking: [{ issue: "引入 find", evidence: EVIDENCE }], notes: "违反 P0" });
|
|
81
|
+
const result = postProcessReview(text);
|
|
75
82
|
assert.equal(result.status, "ok");
|
|
76
83
|
assert.equal(result.verdict, "reject");
|
|
77
84
|
assert.equal(result.blocking[0].evidence.line, 12);
|
|
78
85
|
assert.equal(result.notes, "违反 P0");
|
|
86
|
+
assert.equal(result.summary, SUMMARY);
|
|
87
|
+
assert.deepEqual(result.passed, [], "reject 时没给 passed 就是空数组,不补造");
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test("pass 但没有 passed(只写「通过」两个字)判 invalid:missing-passed", () => {
|
|
91
|
+
const text = JSON.stringify({ verdict: "pass", summary: SUMMARY, blocking: [] });
|
|
92
|
+
const result = postProcessReview(text);
|
|
93
|
+
assert.equal(result.status, "invalid");
|
|
94
|
+
assert.match(result.detail ?? "", /missing-passed/);
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
test("reject / revise 但没有 blocking 判 invalid:missing-blocking", () => {
|
|
98
|
+
for (const verdict of ["reject", "revise"]) {
|
|
99
|
+
const result = postProcessReview(JSON.stringify({ verdict, summary: SUMMARY, blocking: [] }));
|
|
100
|
+
assert.equal(result.status, "invalid", verdict);
|
|
101
|
+
assert.match(result.detail ?? "", /missing-blocking/);
|
|
102
|
+
}
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
test("passed 项也要有 point 与证据:缺了就是 missing-evidence", () => {
|
|
106
|
+
const noEvidence = postProcessReview(JSON.stringify({ verdict: "pass", summary: SUMMARY, blocking: [], passed: [{ point: "没问题" }] }));
|
|
107
|
+
assert.equal(noEvidence.status, "invalid");
|
|
108
|
+
assert.match(noEvidence.detail ?? "", /missing-evidence: passed/);
|
|
109
|
+
assert.match(noEvidence.detail ?? "", /evidence 不是对象/);
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
test("没写 summary(缺字段 / 空串 / 非字符串)判 invalid:bad-summary", () => {
|
|
113
|
+
for (const summary of [undefined, "", " ", 42]) {
|
|
114
|
+
const result = postProcessReview(JSON.stringify({ verdict: "pass", summary, blocking: [], passed: PASSED }));
|
|
115
|
+
assert.equal(result.status, "invalid", `summary=${String(summary)}`);
|
|
116
|
+
assert.match(result.detail ?? "", /bad-summary/);
|
|
117
|
+
}
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
test(`summary 超 ${SUMMARY_MAX} 字按上限截断(含省略号不超过 ${SUMMARY_MAX}),不判 invalid`, () => {
|
|
121
|
+
const long = "因为".repeat(60);
|
|
122
|
+
const result = postProcessReview(JSON.stringify({ verdict: "pass", summary: long, blocking: [], passed: PASSED }));
|
|
123
|
+
assert.equal(result.status, "ok");
|
|
124
|
+
assert.equal(result.summary?.length, SUMMARY_MAX);
|
|
125
|
+
assert.ok(result.summary?.endsWith("…"), "截断要留省略号,让人知道后面还有话");
|
|
79
126
|
});
|
|
80
127
|
|
|
81
128
|
test("机械复验失败一票 reject,覆盖 LLM 的 pass", () => {
|
|
@@ -103,3 +103,12 @@ test("verdict 语义写进 prompt:能改的小毛病不许判 reject(真实
|
|
|
103
103
|
assert.match(prompt, /revise:方向可以接受/);
|
|
104
104
|
assert.match(prompt, /reject:删掉了原有的明确禁止项/);
|
|
105
105
|
});
|
|
106
|
+
|
|
107
|
+
test("输出契约要求写「为什么」:summary 必填 + pass 必须给 passed", () => {
|
|
108
|
+
const prompt = buildReviewPrompt({ skillName: "demo", skillText: "正文", diffApplied: true, diff: "+++ b/x\n", hardRules: [] });
|
|
109
|
+
assert.match(prompt, /"summary":"一句话结论(≤80 字,必须写清因为什么)"/);
|
|
110
|
+
assert.match(prompt, /"passed":\[\{"point"/);
|
|
111
|
+
assert.match(prompt, /summary 必填/);
|
|
112
|
+
assert.match(prompt, /verdict=pass 时 blocking=\[\] 且 passed 至少 1 条/);
|
|
113
|
+
assert.match(prompt, /verdict=reject\/revise 时 blocking 至少 1 条/);
|
|
114
|
+
});
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 评审记录落盘形态的单测:`03-review.md` 必须写出「为什么」(summary + 通过理由),
|
|
3
|
+
* 且 v0.1.9 之前的老记录(没有 summary / passed 字段)读进来也不能抛。
|
|
4
|
+
*
|
|
5
|
+
* 单独一份测试是因为 review-record.ts 全是纯函数;落盘字段与服务端回写由
|
|
6
|
+
* test/review-supervisor.test.ts 覆盖,细节页读取由 test/web-run-detail.test.ts 覆盖。
|
|
7
|
+
*/
|
|
8
|
+
import assert from "node:assert/strict";
|
|
9
|
+
import test from "node:test";
|
|
10
|
+
import { renderReviewMarkdown, type ReviewRecord } from "../lib/review-record.ts";
|
|
11
|
+
|
|
12
|
+
const EVIDENCE = { file: ".pi/skills/demo/SKILL.md", line: 3, quote: "description: 检索日志用 rg" };
|
|
13
|
+
|
|
14
|
+
function record(extra: Partial<ReviewRecord> = {}): ReviewRecord {
|
|
15
|
+
return {
|
|
16
|
+
status: "rejected",
|
|
17
|
+
verdict: "revise",
|
|
18
|
+
blocking: [{ issue: "描述承诺大于正文", evidence: EVIDENCE }],
|
|
19
|
+
reviewerModel: "deepseek/deepseek-flash",
|
|
20
|
+
proposerModel: "deepseek/deepseek-flash",
|
|
21
|
+
sameModel: false,
|
|
22
|
+
action: "forbidden",
|
|
23
|
+
reviewedAt: 1794134158913,
|
|
24
|
+
reviewedAtIso: "2026-11-08T10:35:58.913Z",
|
|
25
|
+
...extra,
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
test("03-review.md:summary 紧跟在标题后,blocking 与 passed 各成一段", () => {
|
|
30
|
+
const md = renderReviewMarkdown(
|
|
31
|
+
record({
|
|
32
|
+
summary: "描述承诺了正文没有的能力,其余约束未被破坏",
|
|
33
|
+
passed: [{ point: "原有禁令仍完整保留", evidence: { ...EVIDENCE, line: 9, quote: "一律用 rg" } }],
|
|
34
|
+
}),
|
|
35
|
+
"demo",
|
|
36
|
+
);
|
|
37
|
+
const lines = md.split("\n");
|
|
38
|
+
assert.equal(lines[0], "# 评审结论:revise");
|
|
39
|
+
assert.equal(lines[2], "**描述承诺了正文没有的能力,其余约束未被破坏**", "摘要要排在元信息之前");
|
|
40
|
+
assert.match(md, /## blocking\n\n- 描述承诺大于正文\n {2}- 证据:\.pi\/skills\/demo\/SKILL\.md:3 「description: 检索日志用 rg」/);
|
|
41
|
+
assert.match(md, /## passed\n\n- 原有禁令仍完整保留\n {2}- 证据:\.pi\/skills\/demo\/SKILL\.md:9 「一律用 rg」/);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
test("03-review.md:老记录(没有 summary / passed)不抛,也不渲染空段", () => {
|
|
45
|
+
const md = renderReviewMarkdown(record(), "demo");
|
|
46
|
+
assert.equal(md.includes("## passed"), false, "没有通过理由就别留空段");
|
|
47
|
+
assert.equal(md.includes("**"), false, "没有摘要就没有加粗行(同模型警告是 > 开头,不受影响)");
|
|
48
|
+
assert.match(md, /## blocking/);
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
test("03-review.md:pass 且没有 blocking 时写「blocking:无」,通过理由照常列出", () => {
|
|
52
|
+
const md = renderReviewMarkdown(
|
|
53
|
+
record({ status: "reviewed", verdict: "pass", action: "auto", blocking: [], summary: "只给 description 补了触发词", passed: [{ point: "原有禁令仍在", evidence: EVIDENCE }] }),
|
|
54
|
+
"demo",
|
|
55
|
+
);
|
|
56
|
+
assert.match(md, /blocking:无/);
|
|
57
|
+
assert.match(md, /## passed/);
|
|
58
|
+
assert.equal(md.includes("## blocking"), false);
|
|
59
|
+
});
|
|
@@ -21,7 +21,14 @@ const DESC_DIFF = [
|
|
|
21
21
|
"-description: 旧描述",
|
|
22
22
|
"+description: 检索日志时用 rg,禁止 find 全盘扫",
|
|
23
23
|
].join("\n");
|
|
24
|
-
const
|
|
24
|
+
const EVIDENCE_ROW = { file: ".pi/skills/demo/SKILL.md", line: 4, quote: "description: 检索日志时用 rg,禁止 find 全盘扫" };
|
|
25
|
+
const PASS = JSON.stringify({
|
|
26
|
+
verdict: "pass",
|
|
27
|
+
summary: "新描述只补了触发场景词,没删原有禁令",
|
|
28
|
+
blocking: [],
|
|
29
|
+
passed: [{ point: "原有禁止项仍在正文里", evidence: EVIDENCE_ROW }],
|
|
30
|
+
notes: "没问题",
|
|
31
|
+
});
|
|
25
32
|
const RUN_JSON = JSON.stringify({ runId: "run-1", status: "dispatched", cwd: "/repo", stages: [{ name: "analyze", model: "deepseek/flash", usage: {}, ms: 10 }] });
|
|
26
33
|
/** 只放一条与仓库同形的 rg-over-find:用例不依赖 .pi/evolution/rules 文件是否在场 */
|
|
27
34
|
const RULES = compileRules({
|
|
@@ -74,11 +81,18 @@ test("正常路径:pass → reviewed,写 03-review.json/.md 并回写 run.js
|
|
|
74
81
|
assert.equal(review.proposerModel, "deepseek/flash");
|
|
75
82
|
assert.equal(review.sameModel, true);
|
|
76
83
|
assert.equal(review.action, "auto");
|
|
84
|
+
// 结论要能解释「为什么」:summary 与 passed 都得落盘(面板详情页读的就是这两个字段)
|
|
85
|
+
assert.equal(review.summary, "新描述只补了触发场景词,没删原有禁令");
|
|
86
|
+
assert.equal((review.passed as unknown[]).length, 1);
|
|
77
87
|
const md = fs.text(`${RUN_DIR}/03-review.md`) ?? "";
|
|
78
88
|
assert.ok(md.startsWith("> 同模型自评"), "同模型自评警告必须在首行");
|
|
89
|
+
assert.match(md, /\*\*新描述只补了触发场景词,没删原有禁令\*\*/);
|
|
90
|
+
assert.match(md, /## passed/);
|
|
79
91
|
const run = fs.json(`${RUN_DIR}/run.json`) as Record<string, unknown>;
|
|
80
92
|
assert.equal(run.status, "reviewed");
|
|
81
93
|
assert.deepEqual((run.review as Record<string, unknown>).sameModel, true);
|
|
94
|
+
assert.equal((run.review as Record<string, unknown>).summary, "新描述只补了触发场景词,没删原有禁令");
|
|
95
|
+
assert.equal((run.review as Record<string, unknown>).passed, 1);
|
|
82
96
|
assert.equal(prompts.length, 0);
|
|
83
97
|
});
|
|
84
98
|
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* 必红自检单测(设计稿 12.4
|
|
2
|
+
* 必红自检单测(设计稿 12.4):三个坏样本必须被拦住,一个正样本必须放行且写出理由;
|
|
3
|
+
* 任何一条不符就算评审器不可信。
|
|
3
4
|
*
|
|
4
5
|
* 假 spawn 按 prompt 里的「技能名」回不同结论,所以用例不受 fixtures 目录枚举顺序影响。
|
|
6
|
+
* 回的 JSON 必须是能过 postProcessReview 的合法形状(summary 必填,pass 必须带 passed),
|
|
7
|
+
* 否则测的就不是「期望命中」而是「载荷校验」了。
|
|
5
8
|
*/
|
|
6
9
|
import { test } from "node:test";
|
|
7
10
|
import assert from "node:assert/strict";
|
|
@@ -20,11 +23,11 @@ function tmpDir(): string {
|
|
|
20
23
|
}
|
|
21
24
|
|
|
22
25
|
/** 按 prompt 末尾的「技能名:x」挑结论 */
|
|
23
|
-
function spawnBySkill(verdicts: Record<string, string
|
|
26
|
+
function spawnBySkill(verdicts: Record<string, string>, opts: { dropPassed?: boolean } = {}): ReviewSpawn {
|
|
24
27
|
return (_command, args) => {
|
|
25
28
|
const prompt = args[3] ?? "";
|
|
26
29
|
const skill = Object.keys(verdicts).find((name) => prompt.includes(`技能名:${name}`));
|
|
27
|
-
const reply =
|
|
30
|
+
const reply = fakeReply(skill ? (verdicts[skill] ?? "pass") : "pass", { dropPassed: opts.dropPassed === true });
|
|
28
31
|
const child = new FakeReviewChild();
|
|
29
32
|
setTimeout(() => {
|
|
30
33
|
child.stdout.emit("data", reply);
|
|
@@ -34,42 +37,74 @@ function spawnBySkill(verdicts: Record<string, string>): ReviewSpawn {
|
|
|
34
37
|
};
|
|
35
38
|
}
|
|
36
39
|
|
|
37
|
-
|
|
40
|
+
const FAKE_EVIDENCE = { file: ".pi/skills/demo/SKILL.md", line: 9, quote: "凡是查日志一律用 rg" };
|
|
41
|
+
|
|
42
|
+
/** 造一份合法评审输出:reject/revise 带 blocking,pass 带 passed(可故意抽掉,测正样本的必红判定) */
|
|
43
|
+
function fakeReply(verdict: string, opts: { dropPassed: boolean }): string {
|
|
44
|
+
const summary = "假评审:只校验自检链路,不代表真实评审质量";
|
|
45
|
+
if (verdict === "pass") {
|
|
46
|
+
const passed = opts.dropPassed ? [] : [{ point: "原有禁令仍完整保留", evidence: FAKE_EVIDENCE }];
|
|
47
|
+
return JSON.stringify({ verdict, summary, blocking: [], passed, notes: "自检" });
|
|
48
|
+
}
|
|
49
|
+
return JSON.stringify({ verdict, summary, blocking: [{ issue: "有问题", evidence: FAKE_EVIDENCE }], passed: [], notes: "自检" });
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
test("三个坏样本 + 一个正样本都能被读到,期望分别是 reject/reject/revise/pass", () => {
|
|
38
53
|
const fixtures = loadFixtures(FIXTURES);
|
|
39
|
-
assert.deepEqual(fixtures.map((item) => item.name).sort(), ["delete-ban", "rg-to-find", "weak-description"]);
|
|
54
|
+
assert.deepEqual(fixtures.map((item) => item.name).sort(), ["delete-ban", "good-description", "rg-to-find", "weak-description"]);
|
|
40
55
|
const expected = Object.fromEntries(fixtures.map((item) => [item.name, item.expected]));
|
|
41
|
-
assert.deepEqual(expected, {
|
|
56
|
+
assert.deepEqual(expected, {
|
|
57
|
+
"delete-ban": "reject",
|
|
58
|
+
"good-description": "pass",
|
|
59
|
+
"rg-to-find": "reject",
|
|
60
|
+
"weak-description": "revise",
|
|
61
|
+
});
|
|
42
62
|
for (const fixture of fixtures) {
|
|
43
63
|
assert.ok(fixture.diff.length > 0, `${fixture.name} 缺 diff`);
|
|
44
64
|
assert.ok(fixture.after.length > 0, `${fixture.name} 缺改后全文`);
|
|
45
65
|
}
|
|
46
66
|
});
|
|
47
67
|
|
|
48
|
-
test("
|
|
68
|
+
test("四样本全部命中期望 → passed:true 并落盘 selftest.json", async () => {
|
|
49
69
|
const dir = tmpDir();
|
|
50
70
|
const report = await runSelftest({
|
|
51
71
|
evolutionDir: dir,
|
|
52
72
|
cwd: dir,
|
|
53
73
|
fixturesDir: FIXTURES,
|
|
54
|
-
spawnFn: spawnBySkill({ "ui-verify-lite": "reject", "rg-retrieval": "reject", "misc-helper": "revise" }),
|
|
55
|
-
pollMs: 1,
|
|
74
|
+
spawnFn: spawnBySkill({ "ui-verify-lite": "reject", "rg-retrieval": "reject", "misc-helper": "revise", "log-helper": "pass" }),
|
|
56
75
|
});
|
|
57
76
|
assert.equal(report.passed, true);
|
|
58
|
-
assert.equal(report.results.length,
|
|
59
|
-
assert.deepEqual(report.results.map((item) => item.actual), ["reject", "reject", "revise"]);
|
|
77
|
+
assert.equal(report.results.length, 4);
|
|
78
|
+
assert.deepEqual(report.results.map((item) => item.actual), ["reject", "pass", "reject", "revise"]);
|
|
60
79
|
const saved = JSON.parse(fs.readFileSync(selftestFile(dir), "utf8")) as { passed: boolean };
|
|
61
80
|
assert.equal(saved.passed, true);
|
|
62
81
|
assert.ok(!renderSelftestReport(report).includes("【红】"));
|
|
63
82
|
});
|
|
64
83
|
|
|
84
|
+
test("正样本判 pass 但没给 passed 理由 → 判不合格并写清差在哪(必红自检要能拉住无理由的通过)", async () => {
|
|
85
|
+
const dir = tmpDir();
|
|
86
|
+
const report = await runSelftest({
|
|
87
|
+
evolutionDir: dir,
|
|
88
|
+
cwd: dir,
|
|
89
|
+
fixturesDir: FIXTURES,
|
|
90
|
+
spawnFn: spawnBySkill({ "ui-verify-lite": "reject", "rg-retrieval": "reject", "misc-helper": "revise" }, { dropPassed: true }),
|
|
91
|
+
});
|
|
92
|
+
assert.equal(report.passed, false);
|
|
93
|
+
const good = report.results.find((item) => item.name === "good-description");
|
|
94
|
+
assert.equal(good?.ok, false);
|
|
95
|
+
assert.equal(good?.actual, "pass(缺 passed 理由)");
|
|
96
|
+
const text = renderSelftestReport(report);
|
|
97
|
+
assert.ok(text.startsWith(`【红】${UNTRUSTED_NOTE}`), "红字必须在首行");
|
|
98
|
+
assert.match(text, /✗ good-description/);
|
|
99
|
+
});
|
|
100
|
+
|
|
65
101
|
test("有一个被 pass → passed:false,报告首行是红字", async () => {
|
|
66
102
|
const dir = tmpDir();
|
|
67
103
|
const report = await runSelftest({
|
|
68
104
|
evolutionDir: dir,
|
|
69
105
|
cwd: dir,
|
|
70
106
|
fixturesDir: FIXTURES,
|
|
71
|
-
spawnFn: spawnBySkill({ "ui-verify-lite": "reject", "rg-retrieval": "reject", "misc-helper": "pass" }),
|
|
72
|
-
pollMs: 1,
|
|
107
|
+
spawnFn: spawnBySkill({ "ui-verify-lite": "reject", "rg-retrieval": "reject", "misc-helper": "pass", "log-helper": "pass" }),
|
|
73
108
|
});
|
|
74
109
|
assert.equal(report.passed, false);
|
|
75
110
|
const text = renderSelftestReport(report);
|
|
@@ -83,11 +118,11 @@ test("单个样本就 pass(其余 reject)也判不可信", async () => {
|
|
|
83
118
|
evolutionDir: dir,
|
|
84
119
|
cwd: dir,
|
|
85
120
|
fixturesDir: FIXTURES,
|
|
86
|
-
spawnFn: spawnBySkill({ "ui-verify-lite": "pass", "rg-retrieval": "reject", "misc-helper": "revise" }),
|
|
87
|
-
pollMs: 1,
|
|
121
|
+
spawnFn: spawnBySkill({ "ui-verify-lite": "pass", "rg-retrieval": "reject", "misc-helper": "revise", "log-helper": "pass" }),
|
|
88
122
|
});
|
|
89
123
|
assert.equal(report.passed, false);
|
|
90
|
-
|
|
124
|
+
// 四个样本里 3 条命中(含正样本)也不算过:只要有一个坏样本被放行就是不可信
|
|
125
|
+
assert.equal(report.results.filter((item) => item.ok).length, 3);
|
|
91
126
|
});
|
|
92
127
|
|
|
93
128
|
test("fixture 缺 change.diff 被跳过,不猜也不补", () => {
|
|
@@ -1,18 +1,19 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* 筛选栏的验收测试(用户 2026-11-08:「这个搜索框的技能和状态不用了吧, 技能不知道用来搜索什么,
|
|
3
|
+
* 状态应该在顶部 tab 包含了吧」)。
|
|
3
4
|
*
|
|
4
|
-
*
|
|
5
|
-
* 1.
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
5
|
+
* 口径:
|
|
6
|
+
* 1. 表单**只剩「时间」**一个条件——技能看不出该填什么、状态与顶部 tab 重复,两个都撤掉;
|
|
7
|
+
* 2. 撤掉的是**输入控件**,不是参数:`?skill=` / `?status=` 仍由 `parseListQuery` 解析,
|
|
8
|
+
* 技能页点进来、旧书签、代码内部跳转都不会断;
|
|
9
|
+
* 3. 表单仍是 GET + 保留 `view`,否则一提交就跳出当前页。
|
|
9
10
|
*
|
|
10
11
|
* 纪律:纯字符串断言,不碰文件系统、不起服务。
|
|
11
12
|
*/
|
|
12
13
|
import { test } from "node:test";
|
|
13
14
|
import assert from "node:assert/strict";
|
|
14
15
|
|
|
15
|
-
import { parseListQuery
|
|
16
|
+
import { parseListQuery } from "../lib/web/filter.ts";
|
|
16
17
|
import { renderFilterForm } from "../lib/web/render-controls.ts";
|
|
17
18
|
|
|
18
19
|
const FILTER = parseListQuery(new URLSearchParams(""));
|
|
@@ -22,36 +23,38 @@ function form(): string {
|
|
|
22
23
|
return renderFilterForm("pending", FILTER, EXTRAS);
|
|
23
24
|
}
|
|
24
25
|
|
|
25
|
-
test("
|
|
26
|
+
test("筛选栏不再有技能输入框与状态下拉", () => {
|
|
26
27
|
const html = form();
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
28
|
+
assert.ok(!html.includes('name="skill"'), "技能搜索框已撤掉");
|
|
29
|
+
assert.ok(!html.includes('name="status"'), "状态下拉已撤掉(顶部 tab 就是状态)");
|
|
30
|
+
assert.ok(!html.includes("全部状态"));
|
|
31
|
+
assert.ok(!html.includes("全部技能"));
|
|
31
32
|
});
|
|
32
33
|
|
|
33
|
-
test("
|
|
34
|
+
test("时间筛选还在,且三个档位齐全", () => {
|
|
34
35
|
const html = form();
|
|
35
|
-
|
|
36
|
-
for (const
|
|
37
|
-
assert.ok(
|
|
36
|
+
assert.ok(html.includes('name="range"'));
|
|
37
|
+
for (const label of ["最近 7 天", "最近 30 天", "全部时间"]) {
|
|
38
|
+
assert.ok(html.includes(`>${label}<`), `缺时间档位:${label}`);
|
|
38
39
|
}
|
|
39
40
|
});
|
|
40
41
|
|
|
41
|
-
test("
|
|
42
|
-
|
|
43
|
-
|
|
42
|
+
test("表单仍是 GET,且保留 view", () => {
|
|
43
|
+
const html = form();
|
|
44
|
+
assert.ok(html.includes('method="get"'));
|
|
45
|
+
assert.ok(html.includes('<input type="hidden" name="view" value="pending">'));
|
|
44
46
|
});
|
|
45
47
|
|
|
46
|
-
test("
|
|
47
|
-
const
|
|
48
|
-
assert.
|
|
49
|
-
assert.
|
|
50
|
-
assert.
|
|
48
|
+
test("参数没被删:URL 里的 skill / status 仍然生效", () => {
|
|
49
|
+
const parsed = parseListQuery(new URLSearchParams("skill=deploy-test&status=stale&range=30d"));
|
|
50
|
+
assert.equal(parsed.skill, "deploy-test");
|
|
51
|
+
assert.equal(parsed.status, "stale");
|
|
52
|
+
assert.equal(parsed.range, "30d");
|
|
51
53
|
});
|
|
52
54
|
|
|
53
|
-
test("
|
|
54
|
-
const
|
|
55
|
-
|
|
56
|
-
assert.ok(html.includes('
|
|
55
|
+
test("技能页跳过来的链接照旧能带 skill 参数(只是不再有输入框)", () => {
|
|
56
|
+
const parsed = parseListQuery(new URLSearchParams("skill=ui-verify"));
|
|
57
|
+
const html = renderFilterForm("reports", parsed, EXTRAS);
|
|
58
|
+
assert.ok(!html.includes('name="skill"'));
|
|
59
|
+
assert.equal(parsed.skill, "ui-verify", "筛选逻辑要能吃到链接里的技能");
|
|
57
60
|
});
|