pi-skill-evolve 0.1.9 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/skill-evolve/fixtures/selftest/good-description/after.md +9 -0
- package/extensions/skill-evolve/fixtures/selftest/good-description/change.diff +13 -0
- package/extensions/skill-evolve/fixtures/selftest/good-description/fixture.json +6 -0
- package/extensions/skill-evolve/lib/replay-fixture.ts +5 -2
- package/extensions/skill-evolve/lib/review-post.ts +92 -21
- package/extensions/skill-evolve/lib/review-prompt.ts +5 -1
- package/extensions/skill-evolve/lib/review-record.ts +14 -1
- package/extensions/skill-evolve/lib/review-supervisor.ts +13 -1
- package/extensions/skill-evolve/lib/runs.ts +4 -0
- package/extensions/skill-evolve/lib/selftest.ts +22 -5
- package/extensions/skill-evolve/lib/web/applied.ts +6 -1
- package/extensions/skill-evolve/lib/web/approvals.ts +7 -0
- package/extensions/skill-evolve/lib/web/done.ts +20 -6
- package/extensions/skill-evolve/lib/web/pending.ts +7 -1
- package/extensions/skill-evolve/lib/web/render-rows.ts +11 -2
- package/extensions/skill-evolve/lib/web/render-run.ts +46 -10
- package/extensions/skill-evolve/lib/web/reports.ts +11 -3
- package/extensions/skill-evolve/lib/web/run-detail.ts +41 -13
- package/extensions/skill-evolve/lib/web/style.ts +2 -0
- package/extensions/skill-evolve/test/review-post.test.ts +53 -6
- package/extensions/skill-evolve/test/review-prompt.test.ts +9 -0
- package/extensions/skill-evolve/test/review-record.test.ts +59 -0
- package/extensions/skill-evolve/test/review-supervisor.test.ts +15 -1
- package/extensions/skill-evolve/test/selftest.test.ts +51 -16
- package/extensions/skill-evolve/test/web-redesign.test.ts +1 -1
- package/extensions/skill-evolve/test/web-render-run.test.ts +70 -1
- package/extensions/skill-evolve/test/web-run-detail.test.ts +27 -0
- package/extensions/skill-evolve/test/web-view-coverage.test.ts +100 -0
- package/package.json +1 -1
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
import * as fs from "node:fs";
|
|
6
6
|
import * as path from "node:path";
|
|
7
7
|
import { readVerify } from "../apply.ts";
|
|
8
|
-
import type { RunSummary } from "../runs.ts";
|
|
8
|
+
import type { RunStatus, RunSummary } from "../runs.ts";
|
|
9
9
|
import { readRunError } from "../stale.ts";
|
|
10
10
|
import { asNumber, asRecord, asText, readJson, reasonLabel } from "./json-read.ts";
|
|
11
11
|
import { sessionFromRecord, type SessionRef } from "./run-session.ts";
|
|
@@ -13,6 +13,12 @@ import { sessionFromRecord, type SessionRef } from "./run-session.ts";
|
|
|
13
13
|
/** 单条报告里最多列几个技能:全量列会把 details 撑成第二份 findings */
|
|
14
14
|
export const MAX_FINDING_ROWS = 5;
|
|
15
15
|
|
|
16
|
+
/**
|
|
17
|
+
* 复盘报告页唯一由 run 状态决定的分支:status=stale 且没有 findings 的失败 run 也要占一行。
|
|
18
|
+
* (其余 run 靠「能读到 01-findings.json」入场,与状态无关)导出给覆盖性测试用。
|
|
19
|
+
*/
|
|
20
|
+
export const REPORTS_RUN_STATUSES: readonly RunStatus[] = ["stale"];
|
|
21
|
+
|
|
16
22
|
export interface FindingRow {
|
|
17
23
|
skill: string;
|
|
18
24
|
kind: string;
|
|
@@ -126,13 +132,15 @@ export function buildReports(evolutionDir: string, runs: readonly RunSummary[]):
|
|
|
126
132
|
// 失败也要有一行档案:每条 run 都能在报告里找到,不能因为没产出就从历史里消失
|
|
127
133
|
const summary = summaries.get(runId);
|
|
128
134
|
const raw = readJson<Record<string, unknown>>(path.join(runDir, "run.json"));
|
|
129
|
-
|
|
135
|
+
const status = summary?.status ?? asText(raw?.status);
|
|
136
|
+
// 判据用清单本体而不是写死 "stale":清单改了代码就跟着改(同类写法见 ./applied.ts)
|
|
137
|
+
if (!REPORTS_RUN_STATUSES.some((known) => known === status)) continue;
|
|
130
138
|
const error = readRunError(runDir);
|
|
131
139
|
const tail = error?.childLogTail ?? [];
|
|
132
140
|
const lastLine = tail.length > 0 ? tail[tail.length - 1]!.trim() : "";
|
|
133
141
|
reports.push({
|
|
134
142
|
runId,
|
|
135
|
-
status
|
|
143
|
+
status,
|
|
136
144
|
trigger: meta.trigger,
|
|
137
145
|
session: meta.session,
|
|
138
146
|
startedAt: summary?.dispatchedAt ?? asNumber(raw?.dispatchedAt),
|
|
@@ -33,6 +33,14 @@ export interface RunBlocking {
|
|
|
33
33
|
quote: string;
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
+
/** 通过理由(03-review.json 的 passed):哪一点没问题 + 支撑它的原文 */
|
|
37
|
+
export interface RunPassing {
|
|
38
|
+
point: string;
|
|
39
|
+
file: string;
|
|
40
|
+
line: number | null;
|
|
41
|
+
quote: string;
|
|
42
|
+
}
|
|
43
|
+
|
|
36
44
|
export interface RunGate {
|
|
37
45
|
gate: string;
|
|
38
46
|
name: string;
|
|
@@ -70,8 +78,18 @@ export interface RunDetail {
|
|
|
70
78
|
/** candidate 目录里的技能名(与 skill 一致才读全文) */
|
|
71
79
|
candidateSkill: string | null;
|
|
72
80
|
candidateText: string | null;
|
|
73
|
-
review: {
|
|
81
|
+
review: {
|
|
82
|
+
status: string;
|
|
83
|
+
verdict: string;
|
|
84
|
+
action: string;
|
|
85
|
+
reviewerModel: string;
|
|
86
|
+
summary: string | null;
|
|
87
|
+
/** 03-review.json 里连 summary / passed 两个键都没有 = v0.1.9 之前写的记录(详情页要说旧版,而不是说「没给理由」) */
|
|
88
|
+
legacy: boolean;
|
|
89
|
+
} | null;
|
|
74
90
|
blocking: RunBlocking[];
|
|
91
|
+
/** 通过理由:旧 run 没有这个字段,读出来是空数组 */
|
|
92
|
+
passed: RunPassing[];
|
|
75
93
|
gates: RunGate[];
|
|
76
94
|
verifyStatus: string | null;
|
|
77
95
|
findings: RunFinding[];
|
|
@@ -95,30 +113,40 @@ function exists(file: string): boolean {
|
|
|
95
113
|
}
|
|
96
114
|
}
|
|
97
115
|
|
|
98
|
-
/** 03-review.json →
|
|
99
|
-
function
|
|
100
|
-
const
|
|
101
|
-
|
|
102
|
-
const blocking: RunBlocking[] = [];
|
|
103
|
-
for (const item of Array.isArray(raw.blocking) ? raw.blocking : []) {
|
|
116
|
+
/** 03-review.json 里的证据项 → 行上的 file / line / quote(blocking 与 passed 共用同一形状) */
|
|
117
|
+
function readEvidenceRows<T>(items: unknown, build: (text: string, file: string, line: number | null, quote: string) => T): T[] {
|
|
118
|
+
const rows: T[] = [];
|
|
119
|
+
for (const item of Array.isArray(items) ? items : []) {
|
|
104
120
|
const record = asRecord(item);
|
|
105
121
|
if (!record) continue;
|
|
106
122
|
const evidence = asRecord(record.evidence);
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
file: asText(evidence?.file),
|
|
110
|
-
line: asNumber(evidence?.line),
|
|
111
|
-
quote: asText(evidence?.quote),
|
|
112
|
-
});
|
|
123
|
+
const text = asText(record.issue) || asText(record.point) || "(未写说明)";
|
|
124
|
+
rows.push(build(text, asText(evidence?.file), asNumber(evidence?.line), asText(evidence?.quote)));
|
|
113
125
|
}
|
|
126
|
+
return rows;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* 03-review.json → verdict / action / reviewerModel / summary + blocking + passed。
|
|
131
|
+
* 旧版评审产出的文件没有 summary / passed:summary 读成 null、passed 读成空数组(不抛、不补造)。
|
|
132
|
+
*/
|
|
133
|
+
function readReview(runDir: string): Pick<RunDetail, "review" | "blocking" | "passed"> {
|
|
134
|
+
const raw = asRecord(readJson<unknown>(path.join(runDir, "03-review.json")));
|
|
135
|
+
if (!raw) return { review: null, blocking: [], passed: [] };
|
|
136
|
+
const blocking = readEvidenceRows<RunBlocking>(raw.blocking, (issue, file, line, quote) => ({ issue, file, line, quote }));
|
|
137
|
+
const passed = readEvidenceRows<RunPassing>(raw.passed, (point, file, line, quote) => ({ point, file, line, quote }));
|
|
114
138
|
return {
|
|
115
139
|
review: {
|
|
116
140
|
status: asText(raw.status, "(未写状态)"),
|
|
117
141
|
verdict: asText(raw.verdict, "(未写 verdict)"),
|
|
118
142
|
action: asText(raw.action, "(未写 action)"),
|
|
119
143
|
reviewerModel: asText(raw.reviewerModel, "(未写模型)"),
|
|
144
|
+
summary: asText(raw.summary, "") || null,
|
|
145
|
+
// 两个键都不存在才算旧记录:新版即使评审失败也会写 passed: [](空数组也是一个键),不能混谈
|
|
146
|
+
legacy: raw.summary === undefined && raw.passed === undefined,
|
|
120
147
|
},
|
|
121
148
|
blocking,
|
|
149
|
+
passed,
|
|
122
150
|
};
|
|
123
151
|
}
|
|
124
152
|
|
|
@@ -113,6 +113,8 @@ a.btn { display: inline-block; color: var(--blue); padding: 1px 8px; border: 1px
|
|
|
113
113
|
.kv .k { color: var(--muted); font-size: 12px }
|
|
114
114
|
.kv .v { color: var(--fg) }
|
|
115
115
|
.kv-line { margin: 4px 0 }
|
|
116
|
+
/* 评审结论摘要:详情页里唯一「大一号」的正文,用户上来先读这一句 */
|
|
117
|
+
.verdict-summary { margin: 6px 0; font-size: 15px; font-weight: 600; color: var(--fg) }
|
|
116
118
|
a.run { color: #c8d3e6; border-bottom: 1px dashed #45505f }
|
|
117
119
|
a.run:hover { border-bottom-color: var(--blue); text-decoration: none }
|
|
118
120
|
a.sess { color: var(--muted); border-bottom: 1px dashed #45505f }
|
|
@@ -1,21 +1,27 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* 评审后处理单测(设计稿 12.3):结构、证据、一致性一律代码判,不信 LLM 自觉。
|
|
3
3
|
* 每条用例对应「模型输出的一种坏法」,覆盖判 invalid / reject 的每一道关。
|
|
4
|
+
* 「结论必须写为什么」也在这里钉住:没有 summary、pass 却没有 passed(或 reject 却没有 blocking)一律 invalid。
|
|
4
5
|
*/
|
|
5
6
|
import { test } from "node:test";
|
|
6
7
|
import assert from "node:assert/strict";
|
|
7
|
-
import { parseReviewJson, postProcessReview } from "../lib/review-post.ts";
|
|
8
|
+
import { parseReviewJson, postProcessReview, SUMMARY_MAX } from "../lib/review-post.ts";
|
|
8
9
|
|
|
9
|
-
const
|
|
10
|
-
const withBlocking = (verdict: string, evidence: unknown): string =>
|
|
11
|
-
JSON.stringify({ verdict, blocking: [{ issue: "有问题", evidence }], notes: "" });
|
|
10
|
+
const SUMMARY = "新描述补了触发场景词,没删原有禁令";
|
|
12
11
|
const EVIDENCE = { file: ".pi/skills/demo/SKILL.md", line: 12, quote: "允许用 find" };
|
|
12
|
+
const PASSED = [{ point: "原有禁止项仍完整保留", evidence: EVIDENCE }];
|
|
13
|
+
const GOOD_PASS = JSON.stringify({ verdict: "pass", summary: SUMMARY, blocking: [], passed: PASSED, notes: "没问题" });
|
|
14
|
+
const withBlocking = (verdict: string, evidence: unknown): string =>
|
|
15
|
+
JSON.stringify({ verdict, summary: SUMMARY, blocking: [{ issue: "有问题", evidence }], passed: [], notes: "" });
|
|
13
16
|
|
|
14
|
-
test("合法 pass 返回 ok", () => {
|
|
17
|
+
test("合法 pass 返回 ok,且 summary / passed 原样带出", () => {
|
|
15
18
|
const result = postProcessReview(GOOD_PASS);
|
|
16
19
|
assert.equal(result.status, "ok");
|
|
17
20
|
assert.equal(result.verdict, "pass");
|
|
18
21
|
assert.deepEqual(result.blocking, []);
|
|
22
|
+
assert.equal(result.summary, SUMMARY);
|
|
23
|
+
assert.equal(result.passed.length, 1);
|
|
24
|
+
assert.equal(result.passed[0].point, "原有禁止项仍完整保留");
|
|
19
25
|
});
|
|
20
26
|
|
|
21
27
|
test("带 ```json 围栏与前后废话也能抠出 JSON", () => {
|
|
@@ -71,11 +77,52 @@ test("pass 却带 blocking 判 invalid:contradiction", () => {
|
|
|
71
77
|
});
|
|
72
78
|
|
|
73
79
|
test("reject 带完整证据返回 ok 且保留 blocking/notes", () => {
|
|
74
|
-
const
|
|
80
|
+
const text = JSON.stringify({ verdict: "reject", summary: SUMMARY, blocking: [{ issue: "引入 find", evidence: EVIDENCE }], notes: "违反 P0" });
|
|
81
|
+
const result = postProcessReview(text);
|
|
75
82
|
assert.equal(result.status, "ok");
|
|
76
83
|
assert.equal(result.verdict, "reject");
|
|
77
84
|
assert.equal(result.blocking[0].evidence.line, 12);
|
|
78
85
|
assert.equal(result.notes, "违反 P0");
|
|
86
|
+
assert.equal(result.summary, SUMMARY);
|
|
87
|
+
assert.deepEqual(result.passed, [], "reject 时没给 passed 就是空数组,不补造");
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test("pass 但没有 passed(只写「通过」两个字)判 invalid:missing-passed", () => {
|
|
91
|
+
const text = JSON.stringify({ verdict: "pass", summary: SUMMARY, blocking: [] });
|
|
92
|
+
const result = postProcessReview(text);
|
|
93
|
+
assert.equal(result.status, "invalid");
|
|
94
|
+
assert.match(result.detail ?? "", /missing-passed/);
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
test("reject / revise 但没有 blocking 判 invalid:missing-blocking", () => {
|
|
98
|
+
for (const verdict of ["reject", "revise"]) {
|
|
99
|
+
const result = postProcessReview(JSON.stringify({ verdict, summary: SUMMARY, blocking: [] }));
|
|
100
|
+
assert.equal(result.status, "invalid", verdict);
|
|
101
|
+
assert.match(result.detail ?? "", /missing-blocking/);
|
|
102
|
+
}
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
test("passed 项也要有 point 与证据:缺了就是 missing-evidence", () => {
|
|
106
|
+
const noEvidence = postProcessReview(JSON.stringify({ verdict: "pass", summary: SUMMARY, blocking: [], passed: [{ point: "没问题" }] }));
|
|
107
|
+
assert.equal(noEvidence.status, "invalid");
|
|
108
|
+
assert.match(noEvidence.detail ?? "", /missing-evidence: passed/);
|
|
109
|
+
assert.match(noEvidence.detail ?? "", /evidence 不是对象/);
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
test("没写 summary(缺字段 / 空串 / 非字符串)判 invalid:bad-summary", () => {
|
|
113
|
+
for (const summary of [undefined, "", " ", 42]) {
|
|
114
|
+
const result = postProcessReview(JSON.stringify({ verdict: "pass", summary, blocking: [], passed: PASSED }));
|
|
115
|
+
assert.equal(result.status, "invalid", `summary=${String(summary)}`);
|
|
116
|
+
assert.match(result.detail ?? "", /bad-summary/);
|
|
117
|
+
}
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
test(`summary 超 ${SUMMARY_MAX} 字按上限截断(含省略号不超过 ${SUMMARY_MAX}),不判 invalid`, () => {
|
|
121
|
+
const long = "因为".repeat(60);
|
|
122
|
+
const result = postProcessReview(JSON.stringify({ verdict: "pass", summary: long, blocking: [], passed: PASSED }));
|
|
123
|
+
assert.equal(result.status, "ok");
|
|
124
|
+
assert.equal(result.summary?.length, SUMMARY_MAX);
|
|
125
|
+
assert.ok(result.summary?.endsWith("…"), "截断要留省略号,让人知道后面还有话");
|
|
79
126
|
});
|
|
80
127
|
|
|
81
128
|
test("机械复验失败一票 reject,覆盖 LLM 的 pass", () => {
|
|
@@ -103,3 +103,12 @@ test("verdict 语义写进 prompt:能改的小毛病不许判 reject(真实
|
|
|
103
103
|
assert.match(prompt, /revise:方向可以接受/);
|
|
104
104
|
assert.match(prompt, /reject:删掉了原有的明确禁止项/);
|
|
105
105
|
});
|
|
106
|
+
|
|
107
|
+
test("输出契约要求写「为什么」:summary 必填 + pass 必须给 passed", () => {
|
|
108
|
+
const prompt = buildReviewPrompt({ skillName: "demo", skillText: "正文", diffApplied: true, diff: "+++ b/x\n", hardRules: [] });
|
|
109
|
+
assert.match(prompt, /"summary":"一句话结论(≤80 字,必须写清因为什么)"/);
|
|
110
|
+
assert.match(prompt, /"passed":\[\{"point"/);
|
|
111
|
+
assert.match(prompt, /summary 必填/);
|
|
112
|
+
assert.match(prompt, /verdict=pass 时 blocking=\[\] 且 passed 至少 1 条/);
|
|
113
|
+
assert.match(prompt, /verdict=reject\/revise 时 blocking 至少 1 条/);
|
|
114
|
+
});
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 评审记录落盘形态的单测:`03-review.md` 必须写出「为什么」(summary + 通过理由),
|
|
3
|
+
* 且 v0.1.9 之前的老记录(没有 summary / passed 字段)读进来也不能抛。
|
|
4
|
+
*
|
|
5
|
+
* 单独一份测试是因为 review-record.ts 全是纯函数;落盘字段与服务端回写由
|
|
6
|
+
* test/review-supervisor.test.ts 覆盖,细节页读取由 test/web-run-detail.test.ts 覆盖。
|
|
7
|
+
*/
|
|
8
|
+
import assert from "node:assert/strict";
|
|
9
|
+
import test from "node:test";
|
|
10
|
+
import { renderReviewMarkdown, type ReviewRecord } from "../lib/review-record.ts";
|
|
11
|
+
|
|
12
|
+
const EVIDENCE = { file: ".pi/skills/demo/SKILL.md", line: 3, quote: "description: 检索日志用 rg" };
|
|
13
|
+
|
|
14
|
+
function record(extra: Partial<ReviewRecord> = {}): ReviewRecord {
|
|
15
|
+
return {
|
|
16
|
+
status: "rejected",
|
|
17
|
+
verdict: "revise",
|
|
18
|
+
blocking: [{ issue: "描述承诺大于正文", evidence: EVIDENCE }],
|
|
19
|
+
reviewerModel: "deepseek/deepseek-flash",
|
|
20
|
+
proposerModel: "deepseek/deepseek-flash",
|
|
21
|
+
sameModel: false,
|
|
22
|
+
action: "forbidden",
|
|
23
|
+
reviewedAt: 1794134158913,
|
|
24
|
+
reviewedAtIso: "2026-11-08T10:35:58.913Z",
|
|
25
|
+
...extra,
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
test("03-review.md:summary 紧跟在标题后,blocking 与 passed 各成一段", () => {
|
|
30
|
+
const md = renderReviewMarkdown(
|
|
31
|
+
record({
|
|
32
|
+
summary: "描述承诺了正文没有的能力,其余约束未被破坏",
|
|
33
|
+
passed: [{ point: "原有禁令仍完整保留", evidence: { ...EVIDENCE, line: 9, quote: "一律用 rg" } }],
|
|
34
|
+
}),
|
|
35
|
+
"demo",
|
|
36
|
+
);
|
|
37
|
+
const lines = md.split("\n");
|
|
38
|
+
assert.equal(lines[0], "# 评审结论:revise");
|
|
39
|
+
assert.equal(lines[2], "**描述承诺了正文没有的能力,其余约束未被破坏**", "摘要要排在元信息之前");
|
|
40
|
+
assert.match(md, /## blocking\n\n- 描述承诺大于正文\n {2}- 证据:\.pi\/skills\/demo\/SKILL\.md:3 「description: 检索日志用 rg」/);
|
|
41
|
+
assert.match(md, /## passed\n\n- 原有禁令仍完整保留\n {2}- 证据:\.pi\/skills\/demo\/SKILL\.md:9 「一律用 rg」/);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
test("03-review.md:老记录(没有 summary / passed)不抛,也不渲染空段", () => {
|
|
45
|
+
const md = renderReviewMarkdown(record(), "demo");
|
|
46
|
+
assert.equal(md.includes("## passed"), false, "没有通过理由就别留空段");
|
|
47
|
+
assert.equal(md.includes("**"), false, "没有摘要就没有加粗行(同模型警告是 > 开头,不受影响)");
|
|
48
|
+
assert.match(md, /## blocking/);
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
test("03-review.md:pass 且没有 blocking 时写「blocking:无」,通过理由照常列出", () => {
|
|
52
|
+
const md = renderReviewMarkdown(
|
|
53
|
+
record({ status: "reviewed", verdict: "pass", action: "auto", blocking: [], summary: "只给 description 补了触发词", passed: [{ point: "原有禁令仍在", evidence: EVIDENCE }] }),
|
|
54
|
+
"demo",
|
|
55
|
+
);
|
|
56
|
+
assert.match(md, /blocking:无/);
|
|
57
|
+
assert.match(md, /## passed/);
|
|
58
|
+
assert.equal(md.includes("## blocking"), false);
|
|
59
|
+
});
|
|
@@ -21,7 +21,14 @@ const DESC_DIFF = [
|
|
|
21
21
|
"-description: 旧描述",
|
|
22
22
|
"+description: 检索日志时用 rg,禁止 find 全盘扫",
|
|
23
23
|
].join("\n");
|
|
24
|
-
const
|
|
24
|
+
const EVIDENCE_ROW = { file: ".pi/skills/demo/SKILL.md", line: 4, quote: "description: 检索日志时用 rg,禁止 find 全盘扫" };
|
|
25
|
+
const PASS = JSON.stringify({
|
|
26
|
+
verdict: "pass",
|
|
27
|
+
summary: "新描述只补了触发场景词,没删原有禁令",
|
|
28
|
+
blocking: [],
|
|
29
|
+
passed: [{ point: "原有禁止项仍在正文里", evidence: EVIDENCE_ROW }],
|
|
30
|
+
notes: "没问题",
|
|
31
|
+
});
|
|
25
32
|
const RUN_JSON = JSON.stringify({ runId: "run-1", status: "dispatched", cwd: "/repo", stages: [{ name: "analyze", model: "deepseek/flash", usage: {}, ms: 10 }] });
|
|
26
33
|
/** 只放一条与仓库同形的 rg-over-find:用例不依赖 .pi/evolution/rules 文件是否在场 */
|
|
27
34
|
const RULES = compileRules({
|
|
@@ -74,11 +81,18 @@ test("正常路径:pass → reviewed,写 03-review.json/.md 并回写 run.js
|
|
|
74
81
|
assert.equal(review.proposerModel, "deepseek/flash");
|
|
75
82
|
assert.equal(review.sameModel, true);
|
|
76
83
|
assert.equal(review.action, "auto");
|
|
84
|
+
// 结论要能解释「为什么」:summary 与 passed 都得落盘(面板详情页读的就是这两个字段)
|
|
85
|
+
assert.equal(review.summary, "新描述只补了触发场景词,没删原有禁令");
|
|
86
|
+
assert.equal((review.passed as unknown[]).length, 1);
|
|
77
87
|
const md = fs.text(`${RUN_DIR}/03-review.md`) ?? "";
|
|
78
88
|
assert.ok(md.startsWith("> 同模型自评"), "同模型自评警告必须在首行");
|
|
89
|
+
assert.match(md, /\*\*新描述只补了触发场景词,没删原有禁令\*\*/);
|
|
90
|
+
assert.match(md, /## passed/);
|
|
79
91
|
const run = fs.json(`${RUN_DIR}/run.json`) as Record<string, unknown>;
|
|
80
92
|
assert.equal(run.status, "reviewed");
|
|
81
93
|
assert.deepEqual((run.review as Record<string, unknown>).sameModel, true);
|
|
94
|
+
assert.equal((run.review as Record<string, unknown>).summary, "新描述只补了触发场景词,没删原有禁令");
|
|
95
|
+
assert.equal((run.review as Record<string, unknown>).passed, 1);
|
|
82
96
|
assert.equal(prompts.length, 0);
|
|
83
97
|
});
|
|
84
98
|
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* 必红自检单测(设计稿 12.4
|
|
2
|
+
* 必红自检单测(设计稿 12.4):三个坏样本必须被拦住,一个正样本必须放行且写出理由;
|
|
3
|
+
* 任何一条不符就算评审器不可信。
|
|
3
4
|
*
|
|
4
5
|
* 假 spawn 按 prompt 里的「技能名」回不同结论,所以用例不受 fixtures 目录枚举顺序影响。
|
|
6
|
+
* 回的 JSON 必须是能过 postProcessReview 的合法形状(summary 必填,pass 必须带 passed),
|
|
7
|
+
* 否则测的就不是「期望命中」而是「载荷校验」了。
|
|
5
8
|
*/
|
|
6
9
|
import { test } from "node:test";
|
|
7
10
|
import assert from "node:assert/strict";
|
|
@@ -20,11 +23,11 @@ function tmpDir(): string {
|
|
|
20
23
|
}
|
|
21
24
|
|
|
22
25
|
/** 按 prompt 末尾的「技能名:x」挑结论 */
|
|
23
|
-
function spawnBySkill(verdicts: Record<string, string
|
|
26
|
+
function spawnBySkill(verdicts: Record<string, string>, opts: { dropPassed?: boolean } = {}): ReviewSpawn {
|
|
24
27
|
return (_command, args) => {
|
|
25
28
|
const prompt = args[3] ?? "";
|
|
26
29
|
const skill = Object.keys(verdicts).find((name) => prompt.includes(`技能名:${name}`));
|
|
27
|
-
const reply =
|
|
30
|
+
const reply = fakeReply(skill ? (verdicts[skill] ?? "pass") : "pass", { dropPassed: opts.dropPassed === true });
|
|
28
31
|
const child = new FakeReviewChild();
|
|
29
32
|
setTimeout(() => {
|
|
30
33
|
child.stdout.emit("data", reply);
|
|
@@ -34,42 +37,74 @@ function spawnBySkill(verdicts: Record<string, string>): ReviewSpawn {
|
|
|
34
37
|
};
|
|
35
38
|
}
|
|
36
39
|
|
|
37
|
-
|
|
40
|
+
const FAKE_EVIDENCE = { file: ".pi/skills/demo/SKILL.md", line: 9, quote: "凡是查日志一律用 rg" };
|
|
41
|
+
|
|
42
|
+
/** 造一份合法评审输出:reject/revise 带 blocking,pass 带 passed(可故意抽掉,测正样本的必红判定) */
|
|
43
|
+
function fakeReply(verdict: string, opts: { dropPassed: boolean }): string {
|
|
44
|
+
const summary = "假评审:只校验自检链路,不代表真实评审质量";
|
|
45
|
+
if (verdict === "pass") {
|
|
46
|
+
const passed = opts.dropPassed ? [] : [{ point: "原有禁令仍完整保留", evidence: FAKE_EVIDENCE }];
|
|
47
|
+
return JSON.stringify({ verdict, summary, blocking: [], passed, notes: "自检" });
|
|
48
|
+
}
|
|
49
|
+
return JSON.stringify({ verdict, summary, blocking: [{ issue: "有问题", evidence: FAKE_EVIDENCE }], passed: [], notes: "自检" });
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
test("三个坏样本 + 一个正样本都能被读到,期望分别是 reject/reject/revise/pass", () => {
|
|
38
53
|
const fixtures = loadFixtures(FIXTURES);
|
|
39
|
-
assert.deepEqual(fixtures.map((item) => item.name).sort(), ["delete-ban", "rg-to-find", "weak-description"]);
|
|
54
|
+
assert.deepEqual(fixtures.map((item) => item.name).sort(), ["delete-ban", "good-description", "rg-to-find", "weak-description"]);
|
|
40
55
|
const expected = Object.fromEntries(fixtures.map((item) => [item.name, item.expected]));
|
|
41
|
-
assert.deepEqual(expected, {
|
|
56
|
+
assert.deepEqual(expected, {
|
|
57
|
+
"delete-ban": "reject",
|
|
58
|
+
"good-description": "pass",
|
|
59
|
+
"rg-to-find": "reject",
|
|
60
|
+
"weak-description": "revise",
|
|
61
|
+
});
|
|
42
62
|
for (const fixture of fixtures) {
|
|
43
63
|
assert.ok(fixture.diff.length > 0, `${fixture.name} 缺 diff`);
|
|
44
64
|
assert.ok(fixture.after.length > 0, `${fixture.name} 缺改后全文`);
|
|
45
65
|
}
|
|
46
66
|
});
|
|
47
67
|
|
|
48
|
-
test("
|
|
68
|
+
test("四样本全部命中期望 → passed:true 并落盘 selftest.json", async () => {
|
|
49
69
|
const dir = tmpDir();
|
|
50
70
|
const report = await runSelftest({
|
|
51
71
|
evolutionDir: dir,
|
|
52
72
|
cwd: dir,
|
|
53
73
|
fixturesDir: FIXTURES,
|
|
54
|
-
spawnFn: spawnBySkill({ "ui-verify-lite": "reject", "rg-retrieval": "reject", "misc-helper": "revise" }),
|
|
55
|
-
pollMs: 1,
|
|
74
|
+
spawnFn: spawnBySkill({ "ui-verify-lite": "reject", "rg-retrieval": "reject", "misc-helper": "revise", "log-helper": "pass" }),
|
|
56
75
|
});
|
|
57
76
|
assert.equal(report.passed, true);
|
|
58
|
-
assert.equal(report.results.length,
|
|
59
|
-
assert.deepEqual(report.results.map((item) => item.actual), ["reject", "reject", "revise"]);
|
|
77
|
+
assert.equal(report.results.length, 4);
|
|
78
|
+
assert.deepEqual(report.results.map((item) => item.actual), ["reject", "pass", "reject", "revise"]);
|
|
60
79
|
const saved = JSON.parse(fs.readFileSync(selftestFile(dir), "utf8")) as { passed: boolean };
|
|
61
80
|
assert.equal(saved.passed, true);
|
|
62
81
|
assert.ok(!renderSelftestReport(report).includes("【红】"));
|
|
63
82
|
});
|
|
64
83
|
|
|
84
|
+
test("正样本判 pass 但没给 passed 理由 → 判不合格并写清差在哪(必红自检要能拉住无理由的通过)", async () => {
|
|
85
|
+
const dir = tmpDir();
|
|
86
|
+
const report = await runSelftest({
|
|
87
|
+
evolutionDir: dir,
|
|
88
|
+
cwd: dir,
|
|
89
|
+
fixturesDir: FIXTURES,
|
|
90
|
+
spawnFn: spawnBySkill({ "ui-verify-lite": "reject", "rg-retrieval": "reject", "misc-helper": "revise" }, { dropPassed: true }),
|
|
91
|
+
});
|
|
92
|
+
assert.equal(report.passed, false);
|
|
93
|
+
const good = report.results.find((item) => item.name === "good-description");
|
|
94
|
+
assert.equal(good?.ok, false);
|
|
95
|
+
assert.equal(good?.actual, "pass(缺 passed 理由)");
|
|
96
|
+
const text = renderSelftestReport(report);
|
|
97
|
+
assert.ok(text.startsWith(`【红】${UNTRUSTED_NOTE}`), "红字必须在首行");
|
|
98
|
+
assert.match(text, /✗ good-description/);
|
|
99
|
+
});
|
|
100
|
+
|
|
65
101
|
test("有一个被 pass → passed:false,报告首行是红字", async () => {
|
|
66
102
|
const dir = tmpDir();
|
|
67
103
|
const report = await runSelftest({
|
|
68
104
|
evolutionDir: dir,
|
|
69
105
|
cwd: dir,
|
|
70
106
|
fixturesDir: FIXTURES,
|
|
71
|
-
spawnFn: spawnBySkill({ "ui-verify-lite": "reject", "rg-retrieval": "reject", "misc-helper": "pass" }),
|
|
72
|
-
pollMs: 1,
|
|
107
|
+
spawnFn: spawnBySkill({ "ui-verify-lite": "reject", "rg-retrieval": "reject", "misc-helper": "pass", "log-helper": "pass" }),
|
|
73
108
|
});
|
|
74
109
|
assert.equal(report.passed, false);
|
|
75
110
|
const text = renderSelftestReport(report);
|
|
@@ -83,11 +118,11 @@ test("单个样本就 pass(其余 reject)也判不可信", async () => {
|
|
|
83
118
|
evolutionDir: dir,
|
|
84
119
|
cwd: dir,
|
|
85
120
|
fixturesDir: FIXTURES,
|
|
86
|
-
spawnFn: spawnBySkill({ "ui-verify-lite": "pass", "rg-retrieval": "reject", "misc-helper": "revise" }),
|
|
87
|
-
pollMs: 1,
|
|
121
|
+
spawnFn: spawnBySkill({ "ui-verify-lite": "pass", "rg-retrieval": "reject", "misc-helper": "revise", "log-helper": "pass" }),
|
|
88
122
|
});
|
|
89
123
|
assert.equal(report.passed, false);
|
|
90
|
-
|
|
124
|
+
// 四个样本里 3 条命中(含正样本)也不算过:只要有一个坏样本被放行就是不可信
|
|
125
|
+
assert.equal(report.results.filter((item) => item.ok).length, 3);
|
|
91
126
|
});
|
|
92
127
|
|
|
93
128
|
test("fixture 缺 change.diff 被跳过,不猜也不补", () => {
|
|
@@ -144,7 +144,7 @@ test("徽章:七种状态各一色(待入队灰 / 待审批蓝 / 复盘中
|
|
|
144
144
|
assert.match(pendingCase("stale", "跑了 10 小时"), /<span class="badge red" title="跑了 10 小时">卡死<\/span>/);
|
|
145
145
|
assert.match(doneCase("applied"), /<span class="badge green" title="">已采纳<\/span>/);
|
|
146
146
|
assert.match(doneCase("rejected"), /<span class="badge darkred" title="">已拒绝<\/span>/);
|
|
147
|
-
assert.match(doneCase("
|
|
147
|
+
assert.match(doneCase("rolled-back"), /<span class="badge faint" title="">已回滚<\/span>/);
|
|
148
148
|
assert.match(doneCase("failed"), /<span class="badge red" title="">失败<\/span>/);
|
|
149
149
|
});
|
|
150
150
|
|
|
@@ -43,7 +43,15 @@ function detail(parts: Partial<RunDetail> = {}): RunDetail {
|
|
|
43
43
|
diff: DIFF_TEXT,
|
|
44
44
|
candidateSkill: "demo",
|
|
45
45
|
candidateText: "---\nname: demo\n---\n# 改后\n",
|
|
46
|
-
|
|
46
|
+
// 默认形态 = 仓库里真实的老 run(354df7db-1794133942):03-review.json 没有 summary / passed
|
|
47
|
+
review: {
|
|
48
|
+
status: "rejected",
|
|
49
|
+
verdict: "revise",
|
|
50
|
+
action: "forbidden",
|
|
51
|
+
reviewerModel: "deepseek/deepseek-flash",
|
|
52
|
+
summary: null,
|
|
53
|
+
legacy: true,
|
|
54
|
+
},
|
|
47
55
|
blocking: [
|
|
48
56
|
{
|
|
49
57
|
issue: "描述承诺大于正文能力",
|
|
@@ -52,6 +60,7 @@ function detail(parts: Partial<RunDetail> = {}): RunDetail {
|
|
|
52
60
|
quote: "<script> 引用",
|
|
53
61
|
},
|
|
54
62
|
],
|
|
63
|
+
passed: [],
|
|
55
64
|
gates: [
|
|
56
65
|
{ gate: "gate0", name: "授权核对", ok: true, detail: "" },
|
|
57
66
|
{ gate: "gate3", name: "复验", ok: false, detail: "复验失败" },
|
|
@@ -101,11 +110,71 @@ test("评审结论:verdict/action/模型 + blocking 表格(file:line 与引
|
|
|
101
110
|
assert.match(html, /verdict <b class="fail">revise<\/b>/);
|
|
102
111
|
assert.match(html, /action <b>forbidden<\/b>/);
|
|
103
112
|
assert.match(html, /<code>deepseek\/deepseek-flash<\/code>/);
|
|
113
|
+
assert.match(html, /<h3>阻断意见<\/h3>/);
|
|
104
114
|
assert.match(html, /<td>描述承诺大于正文能力<\/td><td><code>\.pi\/skills\/demo\/SKILL\.md:3<\/code><\/td>/);
|
|
105
115
|
assert.match(html, /<script> 引用/, "引用原文也要转义");
|
|
106
116
|
assert.equal(html.includes('"blocking"'), false, "blocking 要做成表格,不许把 JSON 直接倒出来");
|
|
107
117
|
});
|
|
108
118
|
|
|
119
|
+
test("评审结论:summary 用大一号的字单独一行,排在 verdict 之后、理由表之前", () => {
|
|
120
|
+
const html = renderRunPage(
|
|
121
|
+
detail({ review: { ...detail().review!, summary: "因为 description 承诺了正文没有的能力", legacy: false } }),
|
|
122
|
+
{ state: state() },
|
|
123
|
+
);
|
|
124
|
+
assert.match(html, /<p class="verdict-summary">因为 description 承诺了正文没有的能力<\/p>/);
|
|
125
|
+
const summaryAt = html.indexOf('class="verdict-summary"');
|
|
126
|
+
assert.ok(summaryAt > html.indexOf('class="kv-line"'), "摘要要在 verdict 那一行之后");
|
|
127
|
+
assert.ok(summaryAt < html.indexOf("<h3>阻断意见</h3>"), "摘要要在理由表之前");
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
test("评审结论:通过时列通过理由表(通过点 + 证据),不写阻断意见段", () => {
|
|
131
|
+
const html = renderRunPage(
|
|
132
|
+
detail({
|
|
133
|
+
status: "reviewed",
|
|
134
|
+
review: { status: "reviewed", verdict: "pass", action: "auto", reviewerModel: "m", summary: "新增条款只约束脚本用法,未碰原有禁令", legacy: false },
|
|
135
|
+
blocking: [],
|
|
136
|
+
passed: [{ point: "没有删除任何明确禁止项", file: "<b>", line: 5, quote: "一律用 rg" }],
|
|
137
|
+
}),
|
|
138
|
+
{ state: state() },
|
|
139
|
+
);
|
|
140
|
+
assert.match(html, /verdict <b class="ok">pass<\/b>/);
|
|
141
|
+
assert.match(html, /<h3>通过理由<\/h3>/);
|
|
142
|
+
assert.match(html, /<th>通过点<\/th>/);
|
|
143
|
+
assert.match(html, /<td>没有删除任何明确禁止项<\/td>/);
|
|
144
|
+
assert.match(html, /<b>:5/, "证据里的尖括号也要转义");
|
|
145
|
+
assert.equal(html.includes("阻断意见"), false, "没有 blocking 就没有阻断意见段");
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
test("评审结论:旧 run 没有 summary / passed 时显示兜底文案,不把空表当结论", () => {
|
|
149
|
+
const html = renderRunPage(detail(), { state: state() });
|
|
150
|
+
assert.match(html, /该 run 由旧版评审产出,没有结论摘要(重跑一次就有)/);
|
|
151
|
+
assert.equal(html.includes("通过理由"), false);
|
|
152
|
+
assert.equal(html.includes("只写了结论,没写理由"), false, "旧 run 已有兜底文案,不再叠一句");
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
test("评审结论:新版评审失败(无 summary,但写了 passed 键)说「没给理由」,不说成旧版", () => {
|
|
156
|
+
const html = renderRunPage(
|
|
157
|
+
detail({
|
|
158
|
+
status: "review_failed",
|
|
159
|
+
review: { status: "review_failed", verdict: "(未写 verdict)", action: "(未写 action)", reviewerModel: "m", summary: null, legacy: false },
|
|
160
|
+
blocking: [],
|
|
161
|
+
passed: [],
|
|
162
|
+
}),
|
|
163
|
+
{ state: state() },
|
|
164
|
+
);
|
|
165
|
+
assert.match(html, /这一轮评审只写了结论,没写理由/);
|
|
166
|
+
assert.equal(html.includes("旧版评审产出"), false, "新版评审失败不能标成旧版");
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
test("评审结论:新版评审两个理由表都空时说「只写了结论,没写理由」,不要说成旧版", () => {
|
|
170
|
+
const html = renderRunPage(
|
|
171
|
+
detail({ review: { ...detail().review!, summary: "跳过评审(配置关闭)", legacy: false }, blocking: [], passed: [] }),
|
|
172
|
+
{ state: state() },
|
|
173
|
+
);
|
|
174
|
+
assert.match(html, /03-review\.json 里没有 blocking \/ passed:这一轮评审只写了结论,没写理由。/);
|
|
175
|
+
assert.equal(html.includes("旧版评审产出"), false);
|
|
176
|
+
});
|
|
177
|
+
|
|
109
178
|
test("五闸:逐条 ok/fail 上色,失败那条带说明", () => {
|
|
110
179
|
const html = renderRunPage(detail(), { state: state() });
|
|
111
180
|
assert.match(html, /<td><code>gate0<\/code><\/td><td>授权核对<\/td><td class="ok">ok<\/td>/);
|
|
@@ -61,6 +61,8 @@ function fullRun(): { evolutionDir: string; runDir: string } {
|
|
|
61
61
|
action: "forbidden",
|
|
62
62
|
reviewerModel: "deepseek/deepseek-flash",
|
|
63
63
|
reviewedAt: NOW + 5_000,
|
|
64
|
+
// 新版契约:结论要带一句「为什么」,通过理由也落盘
|
|
65
|
+
summary: "描述承诺了正文没有的能力",
|
|
64
66
|
notes: "证据不足",
|
|
65
67
|
blocking: [
|
|
66
68
|
{
|
|
@@ -69,6 +71,7 @@ function fullRun(): { evolutionDir: string; runDir: string } {
|
|
|
69
71
|
},
|
|
70
72
|
{ issue: "缺少证据", evidence: {} },
|
|
71
73
|
],
|
|
74
|
+
passed: [{ point: "没有删除原有禁令", evidence: { file: ".pi/skills/demo/SKILL.md", line: 9, quote: "一律用 rg" } }],
|
|
72
75
|
});
|
|
73
76
|
writeJson(path.join(runDir, "04-decision.json"), { status: "rejected", approver: "auto", at: NOW + 6_000 });
|
|
74
77
|
writeJson(path.join(runDir, "06-verify.json"), {
|
|
@@ -133,8 +136,13 @@ test("readRunDetail:六块证据(概要 / diff / 候选全文 / 评审 / 五
|
|
|
133
136
|
verdict: "revise",
|
|
134
137
|
action: "forbidden",
|
|
135
138
|
reviewerModel: "deepseek/deepseek-flash",
|
|
139
|
+
summary: "描述承诺了正文没有的能力",
|
|
140
|
+
legacy: false,
|
|
136
141
|
});
|
|
137
142
|
assert.equal(detail.blocking.length, 2);
|
|
143
|
+
assert.equal(detail.passed.length, 1, "通过理由要能读出来(revise 也可能有没问题的点)");
|
|
144
|
+
assert.equal(detail.passed[0]!.point, "没有删除原有禁令");
|
|
145
|
+
assert.equal(detail.passed[0]!.line, 9);
|
|
138
146
|
assert.equal(detail.blocking[0]!.line, 3, "证据行号要能直接跳过去");
|
|
139
147
|
assert.match(detail.blocking[1]!.issue, /缺少证据/);
|
|
140
148
|
assert.equal(detail.blocking[1]!.file, "", "缺 evidence 的文件位置不能编一个");
|
|
@@ -154,6 +162,25 @@ test("readRunDetail:六块证据(概要 / diff / 候选全文 / 评审 / 五
|
|
|
154
162
|
assert.equal(detail.notes, "只差两处");
|
|
155
163
|
});
|
|
156
164
|
|
|
165
|
+
test("readRunDetail:旧 run 的 03-review.json 没有 summary / passed 时不抛,读成 null / 空数组", () => {
|
|
166
|
+
// 真实样本:.pi/evolution/runs/354df7db-1794133942(v0.1.9 之前产出的评审记录)
|
|
167
|
+
const { evolutionDir, runDir } = workspace();
|
|
168
|
+
writeJson(path.join(runDir, "run.json"), { runId: RUN_ID, status: "rejected", trigger: "auto", dispatchedAt: NOW });
|
|
169
|
+
writeJson(path.join(runDir, "03-review.json"), {
|
|
170
|
+
status: "rejected",
|
|
171
|
+
verdict: "revise",
|
|
172
|
+
action: "forbidden",
|
|
173
|
+
reviewerModel: "deepseek/deepseek-flash",
|
|
174
|
+
notes: "旧版只有 notes",
|
|
175
|
+
blocking: [{ issue: "旧版意见", evidence: { file: "a.md", line: 1, quote: "x" } }],
|
|
176
|
+
});
|
|
177
|
+
const detail = readRunDetail(evolutionDir, RUN_ID)!;
|
|
178
|
+
assert.equal(detail.review?.summary, null, "缺字段就是 null,不编也不抛");
|
|
179
|
+
assert.equal(detail.review?.legacy, true, "两个键都没有才算旧记录,详情页要据此说旧版");
|
|
180
|
+
assert.deepEqual(detail.passed, []);
|
|
181
|
+
assert.equal(detail.blocking.length, 1);
|
|
182
|
+
});
|
|
183
|
+
|
|
157
184
|
test("readRunDetail:runId 不合法(../、绝对路径、子路径)一律 null,不读盘越界", () => {
|
|
158
185
|
const { evolutionDir } = fullRun();
|
|
159
186
|
for (const runId of ["../x", "../../etc", "/etc/passwd", "a/b", "", "."]) {
|