portable-agent-layer 0.108.0 → 0.110.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/assets/skills/scientific-research/SKILL.md +58 -12
- package/package.json +6 -2
- package/src/cli/correction.ts +115 -0
- package/src/cli/infer.ts +1 -1
- package/src/cli/rule.ts +121 -14
- package/src/cli/tree.ts +2 -0
- package/src/hooks/handlers/rating.ts +21 -2
- package/src/hooks/handlers/update-check.ts +6 -1
- package/src/hooks/lib/adaptation-candidates.ts +41 -13
- package/src/hooks/lib/adaptation-rules.ts +138 -9
- package/src/hooks/lib/adaptation-steering.ts +7 -2
- package/src/hooks/lib/adaptation-turns.ts +43 -1
- package/src/hooks/lib/context.ts +2 -2
- package/src/hooks/lib/correction-backfill.ts +56 -0
- package/src/hooks/lib/correction-kind.ts +165 -0
- package/src/hooks/lib/correction-rates.ts +151 -0
- package/src/hooks/lib/inference.ts +43 -18
- package/src/hooks/lib/models.ts +76 -28
- package/src/hooks/lib/opinion-gate.ts +73 -0
- package/src/hooks/lib/rule-drafter.ts +57 -24
- package/src/hooks/lib/semi-static.ts +2 -2
- package/src/tools/control-room/relationship.ts +43 -18
- package/src/tools/control-room/server.ts +12 -5
- package/src/tools/control-room/ui/dist/assets/{index-mFpXpiBj.js → index-D0pEd7Ln.js} +1 -1
- package/src/tools/control-room/ui/dist/index.html +1 -1
- package/src/tools/control-room/ui/lib/write.ts +3 -1
- package/src/tools/control-room/ui/screens/relationship.tsx +101 -29
|
@@ -1,15 +1,24 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Rule candidates: drafts the small model wrote from repeated corrections, each
|
|
3
|
-
* with the proof of its trigger.
|
|
4
|
-
*
|
|
5
|
-
*
|
|
3
|
+
* with the proof of its trigger. A proven candidate becomes a draft rule; one
|
|
4
|
+
* whose trigger failed becomes a draft that can only be kept as an opinion, since
|
|
5
|
+
* the mistake recurs even when no pattern catches it, but only once its kind has
|
|
6
|
+
* recurred enough to measure. One the log is still too short to judge waits and
|
|
7
|
+
* is proven again as the log grows.
|
|
6
8
|
*/
|
|
7
9
|
|
|
8
10
|
import { randomUUID } from "node:crypto";
|
|
9
11
|
import { existsSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
10
12
|
import { resolve } from "node:path";
|
|
11
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
addDraft,
|
|
15
|
+
type DraftInput,
|
|
16
|
+
type OpinionGate,
|
|
17
|
+
readRules,
|
|
18
|
+
type TriggerProof,
|
|
19
|
+
} from "./adaptation-rules";
|
|
12
20
|
import type { RequestedTurn } from "./adaptation-turns";
|
|
21
|
+
import { opinionGate } from "./opinion-gate";
|
|
13
22
|
import { paths } from "./paths";
|
|
14
23
|
import {
|
|
15
24
|
type ProofVerdict,
|
|
@@ -22,12 +31,10 @@ export interface CandidateInput extends DraftInput {
|
|
|
22
31
|
check: string;
|
|
23
32
|
}
|
|
24
33
|
|
|
25
|
-
export
|
|
34
|
+
export type Candidate = CandidateInput & {
|
|
26
35
|
id: string;
|
|
27
36
|
createdAt: string;
|
|
28
|
-
|
|
29
|
-
proof: TriggerProof;
|
|
30
|
-
}
|
|
37
|
+
} & ({ verdict: ProofVerdict; proof: TriggerProof } | { verdict: "untriggered" });
|
|
31
38
|
|
|
32
39
|
function candidatesPath(): string {
|
|
33
40
|
return resolve(paths.adaptation(), "candidates.jsonl");
|
|
@@ -59,15 +66,35 @@ function writeCandidates(candidates: Candidate[]): void {
|
|
|
59
66
|
}
|
|
60
67
|
|
|
61
68
|
function prove(candidate: Candidate, turns: RequestedTurn[]): Candidate {
|
|
69
|
+
if (!candidate.trigger) return { ...candidate, verdict: "untriggered" };
|
|
62
70
|
const proof = proveTrigger(candidate.trigger, turns);
|
|
63
71
|
const verdict = candidate.widens ? widenedProofVerdict(proof) : proofVerdict(proof);
|
|
64
72
|
return { ...candidate, proof, verdict };
|
|
65
73
|
}
|
|
66
74
|
|
|
67
|
-
function
|
|
68
|
-
|
|
75
|
+
function canOnlyBeOpinion(candidate: Candidate): boolean {
|
|
76
|
+
const unproven = candidate.verdict === "failed" || candidate.verdict === "untriggered";
|
|
77
|
+
return unproven && !candidate.widens;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function gateFor(candidate: Candidate, turns: RequestedTurn[]): OpinionGate | undefined {
|
|
81
|
+
if (!candidate.kind || candidate.widens) return undefined;
|
|
82
|
+
return opinionGate(candidate.kind, readRules(), turns);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function draftOf(candidate: Candidate, turns: RequestedTurn[]): DraftInput | null {
|
|
69
86
|
const { id: _id, createdAt: _createdAt, verdict: _verdict, ...draft } = candidate;
|
|
70
|
-
|
|
87
|
+
const gate = gateFor(candidate, turns);
|
|
88
|
+
const gated = gate ? { ...draft, opinionGate: gate } : draft;
|
|
89
|
+
if (!canOnlyBeOpinion(candidate)) return gated;
|
|
90
|
+
if (!candidate.kind || (gate && "corrections" in gate)) return null;
|
|
91
|
+
return { ...gated, offer: "opinion-only" };
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function promoteIfDecided(candidate: Candidate, turns: RequestedTurn[]): void {
|
|
95
|
+
if (candidate.verdict !== "passed" && !canOnlyBeOpinion(candidate)) return;
|
|
96
|
+
const draft = draftOf(candidate, turns);
|
|
97
|
+
if (draft) addDraft(draft);
|
|
71
98
|
}
|
|
72
99
|
|
|
73
100
|
export function recordCandidate(
|
|
@@ -82,7 +109,7 @@ export function recordCandidate(
|
|
|
82
109
|
} as Candidate;
|
|
83
110
|
const candidate = prove(fresh, turns);
|
|
84
111
|
writeCandidates([...readCandidates(), candidate]);
|
|
85
|
-
|
|
112
|
+
promoteIfDecided(candidate, turns);
|
|
86
113
|
return candidate;
|
|
87
114
|
}
|
|
88
115
|
|
|
@@ -91,5 +118,6 @@ export function reproveWaiting(turns: RequestedTurn[]): void {
|
|
|
91
118
|
if (!candidates.some((c) => c.verdict === "waiting")) return;
|
|
92
119
|
const updated = candidates.map((c) => (c.verdict === "waiting" ? prove(c, turns) : c));
|
|
93
120
|
writeCandidates(updated);
|
|
94
|
-
updated.filter((
|
|
121
|
+
for (const candidate of updated.filter((_c, i) => candidates[i].verdict === "waiting"))
|
|
122
|
+
promoteIfDecided(candidate, turns);
|
|
95
123
|
}
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
* Adaptation rules: "when this happens, steer like this", learned from the
|
|
3
3
|
* user's corrections. A rule starts as a draft and only steers once the user
|
|
4
4
|
* approves it. A denied rule stays on record so the same rule is not drafted again.
|
|
5
|
+
* An opinion is an approved rule without a trigger: it steers every session, so
|
|
6
|
+
* the opinions together are capped, and a new one past the cap is refused.
|
|
5
7
|
*/
|
|
6
8
|
|
|
7
9
|
import { randomUUID } from "node:crypto";
|
|
@@ -9,8 +11,11 @@ import { existsSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
|
9
11
|
import { resolve } from "node:path";
|
|
10
12
|
import { paths } from "./paths";
|
|
11
13
|
|
|
12
|
-
|
|
13
|
-
type Decision =
|
|
14
|
+
type RuleStatus = "draft" | "approved" | "denied" | "retired";
|
|
15
|
+
type Decision = "approved" | "denied" | "opinion";
|
|
16
|
+
|
|
17
|
+
const OPINION_CAP = 8;
|
|
18
|
+
const OPINION_CHARS = 1500;
|
|
14
19
|
|
|
15
20
|
export interface RuleTrigger {
|
|
16
21
|
/** Whether the pattern is matched against the user's prompt or the assistant's reply. */
|
|
@@ -28,21 +33,78 @@ export interface TriggerProof {
|
|
|
28
33
|
|
|
29
34
|
export interface DraftInput {
|
|
30
35
|
when: string;
|
|
31
|
-
|
|
36
|
+
/** Absent when the situation shows in no text a pattern could match; it can only be an opinion. */
|
|
37
|
+
trigger?: RuleTrigger;
|
|
32
38
|
steering: string;
|
|
39
|
+
/** The correction kind the rule prevents; its rate measures the rule. */
|
|
40
|
+
kind?: string;
|
|
33
41
|
evidence: string[];
|
|
34
42
|
/** How to tell from a reply whether the steering was followed. */
|
|
35
43
|
check?: string;
|
|
36
44
|
proof?: TriggerProof;
|
|
37
45
|
/** The approved rule whose trigger this draft replaces with a wider one. */
|
|
38
46
|
widens?: string;
|
|
47
|
+
/** Its trigger failed the proof, so it can only be kept as an opinion or denied. */
|
|
48
|
+
offer?: "opinion-only";
|
|
49
|
+
/** Why it cannot be kept as an opinion: too few corrections of its kind, or another targets it. */
|
|
50
|
+
opinionGate?: OpinionGate;
|
|
39
51
|
}
|
|
40
52
|
|
|
53
|
+
export type OpinionGate = { corrections: number } | { duplicates: string };
|
|
54
|
+
|
|
41
55
|
export interface AdaptationRule extends DraftInput {
|
|
42
56
|
id: string;
|
|
43
57
|
status: RuleStatus;
|
|
58
|
+
/** "always" makes an approved rule an opinion, injected every session. */
|
|
59
|
+
delivery?: "always";
|
|
44
60
|
createdAt: string;
|
|
45
61
|
decidedAt?: string;
|
|
62
|
+
retiredAt?: string;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export function isOpinion(rule: AdaptationRule): boolean {
|
|
66
|
+
return rule.status === "approved" && rule.delivery === "always";
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
type Deliverable = Pick<AdaptationRule, "delivery" | "trigger"> & { status: string };
|
|
70
|
+
|
|
71
|
+
export function isTriggerRule<R extends Deliverable>(
|
|
72
|
+
rule: R
|
|
73
|
+
): rule is R & { trigger: RuleTrigger } {
|
|
74
|
+
return rule.status === "approved" && rule.delivery !== "always" && !!rule.trigger;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function opinionLine(rule: Pick<AdaptationRule, "when" | "steering">): string {
|
|
78
|
+
return `- ${rule.when}: ${rule.steering}`;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export function opinionsContext(): string {
|
|
82
|
+
const opinions = readRules().filter(isOpinion);
|
|
83
|
+
if (opinions.length === 0) return "";
|
|
84
|
+
return [
|
|
85
|
+
"## Opinions",
|
|
86
|
+
"The user approved these from past corrections. Follow them on every turn:",
|
|
87
|
+
...opinions.map(opinionLine),
|
|
88
|
+
].join("\n");
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function sameKindOpinion(rules: AdaptationRule[], added: AdaptationRule): string | null {
|
|
92
|
+
const taken = added.kind && rules.find((r) => isOpinion(r) && r.kind === added.kind);
|
|
93
|
+
return taken
|
|
94
|
+
? `Opinion ${taken.id} already targets ${added.kind}, and two would share one rate; retire it first`
|
|
95
|
+
: null;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function opinionRefusal(rules: AdaptationRule[], added: AdaptationRule): string | null {
|
|
99
|
+
const opinions = [...rules.filter(isOpinion), added];
|
|
100
|
+
const sameKind = sameKindOpinion(rules, added);
|
|
101
|
+
if (sameKind) return sameKind;
|
|
102
|
+
if (opinions.length > OPINION_CAP)
|
|
103
|
+
return `Opinions are capped at ${OPINION_CAP}; retire one first`;
|
|
104
|
+
const size = opinions.map(opinionLine).join("\n").length;
|
|
105
|
+
if (size > OPINION_CHARS)
|
|
106
|
+
return `Opinions are capped at ${OPINION_CHARS} characters, this one would make ${size}; retire one first`;
|
|
107
|
+
return null;
|
|
46
108
|
}
|
|
47
109
|
|
|
48
110
|
type DecideResult = { ok: true; rule: AdaptationRule } | { ok: false; reason: string };
|
|
@@ -101,18 +163,85 @@ function approveWidening(
|
|
|
101
163
|
return { ok: true, rule: widened };
|
|
102
164
|
}
|
|
103
165
|
|
|
166
|
+
function decided(rule: AdaptationRule, decision: Decision): AdaptationRule {
|
|
167
|
+
const decidedAt = new Date().toISOString();
|
|
168
|
+
if (decision === "opinion")
|
|
169
|
+
return { ...rule, status: "approved", delivery: "always", decidedAt };
|
|
170
|
+
return { ...rule, status: decision, decidedAt };
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function replaceRule(rules: AdaptationRule[], rule: AdaptationRule): DecideResult {
|
|
174
|
+
writeRules(rules.map((r) => (r.id === rule.id ? rule : r)));
|
|
175
|
+
return { ok: true, rule };
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function gateReason(gate: OpinionGate): string {
|
|
179
|
+
return "corrections" in gate
|
|
180
|
+
? `its kind has only ${gate.corrections} corrections`
|
|
181
|
+
: `${gate.duplicates} already targets its kind`;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function decisionRefusal(rule: AdaptationRule, decision: Decision): string | null {
|
|
185
|
+
if (decision === "opinion" && rule.widens)
|
|
186
|
+
return `Rule ${rule.id} widens a trigger; it cannot be an opinion`;
|
|
187
|
+
if (decision === "opinion" && !rule.kind)
|
|
188
|
+
return `Rule ${rule.id} has no correction kind, so its effect could not be measured`;
|
|
189
|
+
if (decision === "opinion" && rule.opinionGate)
|
|
190
|
+
return `Rule ${rule.id} cannot be an opinion: ${gateReason(rule.opinionGate)}`;
|
|
191
|
+
if (decision === "approved" && !rule.trigger)
|
|
192
|
+
return `Rule ${rule.id} has no trigger; keep it as an opinion or deny it`;
|
|
193
|
+
if (decision === "approved" && rule.offer === "opinion-only")
|
|
194
|
+
return `Rule ${rule.id}'s trigger did not prove; keep it as an opinion or deny it`;
|
|
195
|
+
return null;
|
|
196
|
+
}
|
|
197
|
+
|
|
104
198
|
export function decideRule(id: string, decision: Decision): DecideResult {
|
|
105
199
|
const rules = readRules();
|
|
106
200
|
const rule = rules.find((r) => r.id === id);
|
|
107
201
|
if (!rule) return { ok: false, reason: `No rule with id ${id}` };
|
|
108
202
|
if (rule.status !== "draft")
|
|
109
203
|
return { ok: false, reason: `Rule ${id} was already ${rule.status}` };
|
|
204
|
+
const refused = decisionRefusal(rule, decision);
|
|
205
|
+
if (refused) return { ok: false, reason: refused };
|
|
110
206
|
if (rule.widens && decision === "approved") return approveWidening(rules, rule);
|
|
111
|
-
const
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
207
|
+
const result = decided(rule, decision);
|
|
208
|
+
const refusal = isOpinion(result) ? opinionRefusal(rules, result) : null;
|
|
209
|
+
return refusal ? { ok: false, reason: refusal } : replaceRule(rules, result);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
export interface OpinionInput {
|
|
213
|
+
when: string;
|
|
214
|
+
steering: string;
|
|
215
|
+
kind: string;
|
|
216
|
+
evidence: string[];
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
export function addOpinion(input: OpinionInput): DecideResult {
|
|
220
|
+
const now = new Date().toISOString();
|
|
221
|
+
const opinion: AdaptationRule = {
|
|
222
|
+
...input,
|
|
223
|
+
id: randomUUID().slice(0, 8),
|
|
224
|
+
status: "approved",
|
|
225
|
+
delivery: "always",
|
|
226
|
+
createdAt: now,
|
|
227
|
+
decidedAt: now,
|
|
115
228
|
};
|
|
116
|
-
|
|
117
|
-
|
|
229
|
+
const rules = readRules();
|
|
230
|
+
const refusal = opinionRefusal(rules, opinion);
|
|
231
|
+
if (refusal) return { ok: false, reason: refusal };
|
|
232
|
+
writeRules([...rules, opinion]);
|
|
233
|
+
return { ok: true, rule: opinion };
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
export function retireRule(id: string): DecideResult {
|
|
237
|
+
const rules = readRules();
|
|
238
|
+
const rule = rules.find((r) => r.id === id);
|
|
239
|
+
if (!rule) return { ok: false, reason: `No rule with id ${id}` };
|
|
240
|
+
if (rule.status !== "approved")
|
|
241
|
+
return { ok: false, reason: `Rule ${id} is ${rule.status}, not approved` };
|
|
242
|
+
return replaceRule(rules, {
|
|
243
|
+
...rule,
|
|
244
|
+
status: "retired",
|
|
245
|
+
retiredAt: new Date().toISOString(),
|
|
246
|
+
});
|
|
118
247
|
}
|
|
@@ -5,7 +5,12 @@
|
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import { appendFileSync } from "node:fs";
|
|
8
|
-
import {
|
|
8
|
+
import {
|
|
9
|
+
type AdaptationRule,
|
|
10
|
+
isTriggerRule,
|
|
11
|
+
type RuleTrigger,
|
|
12
|
+
readRules,
|
|
13
|
+
} from "./adaptation-rules";
|
|
9
14
|
import { alreadySentBack } from "./claim-log";
|
|
10
15
|
import { hookFinalReply, hookSessionId } from "./hook-turn";
|
|
11
16
|
import { replyEnd } from "./interaction-samples";
|
|
@@ -20,7 +25,7 @@ const REMINDER_HEADER =
|
|
|
20
25
|
type Side = RuleTrigger["side"];
|
|
21
26
|
|
|
22
27
|
function triggers(rule: AdaptationRule, side: Side, text: string): boolean {
|
|
23
|
-
if (rule
|
|
28
|
+
if (!isTriggerRule(rule) || rule.trigger.side !== side) return false;
|
|
24
29
|
try {
|
|
25
30
|
return new RegExp(rule.trigger.pattern, "i").test(text);
|
|
26
31
|
} catch {
|
|
@@ -6,7 +6,15 @@
|
|
|
6
6
|
* other's lines; a month is deleted whole once it ends before the window.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
-
import {
|
|
9
|
+
import {
|
|
10
|
+
appendFileSync,
|
|
11
|
+
existsSync,
|
|
12
|
+
readdirSync,
|
|
13
|
+
readFileSync,
|
|
14
|
+
renameSync,
|
|
15
|
+
rmSync,
|
|
16
|
+
writeFileSync,
|
|
17
|
+
} from "node:fs";
|
|
10
18
|
import { resolve } from "node:path";
|
|
11
19
|
import { paths } from "./paths";
|
|
12
20
|
|
|
@@ -24,6 +32,15 @@ export interface TurnInput {
|
|
|
24
32
|
issue: string;
|
|
25
33
|
/** Set on a correction: whether the second label agreed. */
|
|
26
34
|
confirmed?: boolean;
|
|
35
|
+
/** Set on a confirmed correction: its kind, under the kind set's version. */
|
|
36
|
+
correctionKind?: StoredCorrectionKind;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
interface StoredCorrectionKind {
|
|
40
|
+
kind: string;
|
|
41
|
+
suggested: string;
|
|
42
|
+
description: string;
|
|
43
|
+
version: number;
|
|
27
44
|
}
|
|
28
45
|
|
|
29
46
|
export interface Turn extends TurnInput {
|
|
@@ -99,6 +116,31 @@ function parseLine(line: string): Turn | null {
|
|
|
99
116
|
}
|
|
100
117
|
}
|
|
101
118
|
|
|
119
|
+
export function turnKey(turn: Turn): string {
|
|
120
|
+
return `${turn.session} ${turn.ts}`;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function replacedLine(line: string, replacements: Map<string, Turn>): string {
|
|
124
|
+
const turn = parseLine(line);
|
|
125
|
+
const replacement = turn && replacements.get(turnKey(turn));
|
|
126
|
+
return replacement ? JSON.stringify(replacement) : line;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Rewrites the given turns in place. Each month file is read and replaced in one
|
|
131
|
+
* synchronous step, leaving only microseconds in which another session's append
|
|
132
|
+
* could be lost; a backfill runs once, so that window is accepted.
|
|
133
|
+
*/
|
|
134
|
+
export function replaceTurns(replacements: Map<string, Turn>): void {
|
|
135
|
+
for (const file of monthFiles()) {
|
|
136
|
+
const path = resolve(paths.adaptation(), file);
|
|
137
|
+
const lines = readFileSync(path, "utf-8").split("\n");
|
|
138
|
+
const rewritten = lines.map((line) => replacedLine(line, replacements));
|
|
139
|
+
writeFileSync(`${path}.tmp`, rewritten.join("\n"));
|
|
140
|
+
renameSync(`${path}.tmp`, path);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
102
144
|
export function readTurns(days: number = WINDOW_DAYS, now: Date = new Date()): Turn[] {
|
|
103
145
|
const cutoff = now.getTime() - days * DAY_MS;
|
|
104
146
|
return monthFiles()
|
package/src/hooks/lib/context.ts
CHANGED
|
@@ -5,11 +5,11 @@
|
|
|
5
5
|
|
|
6
6
|
import { existsSync, readFileSync } from "node:fs";
|
|
7
7
|
import { resolve } from "node:path";
|
|
8
|
+
import { opinionsContext } from "./adaptation-rules";
|
|
8
9
|
import type { AgentName } from "./agent-registry";
|
|
9
10
|
import { resolveAnchor } from "./anchor";
|
|
10
11
|
import { dueNudgeSections } from "./daily-nudge";
|
|
11
12
|
import { loadHandoffContext } from "./handoff-context";
|
|
12
|
-
import { loadOpinionContext } from "./opinions";
|
|
13
13
|
import { paths, toPath } from "./paths";
|
|
14
14
|
import { loadActiveProjectsContext } from "./projects";
|
|
15
15
|
import { loadRecentNotes } from "./relationship";
|
|
@@ -215,7 +215,7 @@ export function buildSystemReminder(
|
|
|
215
215
|
const failures =
|
|
216
216
|
settings.isEnabled("failurePatterns") && !skipSemiStatic ? loadFailurePatterns() : "";
|
|
217
217
|
const opinions =
|
|
218
|
-
!skipSemiStatic && settings.isEnabled("opinions") ?
|
|
218
|
+
!skipSemiStatic && settings.isEnabled("opinions") ? opinionsContext() : "";
|
|
219
219
|
const selfModel =
|
|
220
220
|
!skipSemiStatic && settings.isEnabled("selfModel") ? loadSelfModel() : "";
|
|
221
221
|
const steering = skipSemiStatic ? "" : loadSteeringRules();
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Labels the confirmed corrections that carry no kind of the current version, so
|
|
3
|
+
* the rate of each kind has one labelling across its history. A relabel runs over
|
|
4
|
+
* every confirmed correction again.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { readTurns, replaceTurns, type Turn, turnKey } from "./adaptation-turns";
|
|
8
|
+
import {
|
|
9
|
+
type CorrectionKind,
|
|
10
|
+
correctionKindRequest,
|
|
11
|
+
hasCurrentKind,
|
|
12
|
+
parseCorrectionKind,
|
|
13
|
+
withCorrectionKind,
|
|
14
|
+
} from "./correction-kind";
|
|
15
|
+
import { inference } from "./inference";
|
|
16
|
+
|
|
17
|
+
export type KindLabeller = (turn: Turn) => Promise<CorrectionKind | null>;
|
|
18
|
+
|
|
19
|
+
export interface BackfillResult {
|
|
20
|
+
labelled: number;
|
|
21
|
+
failed: number;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export interface BackfillOptions {
|
|
25
|
+
label?: KindLabeller;
|
|
26
|
+
relabel?: boolean;
|
|
27
|
+
onLabelled?: (turn: Turn, kind: CorrectionKind | null) => void;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function correctionsToLabel(turns: Turn[], relabel = false): Turn[] {
|
|
31
|
+
return turns.filter(
|
|
32
|
+
(turn) => turn.confirmed === true && (relabel || !hasCurrentKind(turn))
|
|
33
|
+
);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export async function labelWithInference(turn: Turn): Promise<CorrectionKind | null> {
|
|
37
|
+
const request = correctionKindRequest(turn.replyEnd, turn.message, turn.issue);
|
|
38
|
+
const result = await inference(request);
|
|
39
|
+
return result.success ? parseCorrectionKind(result.output) : null;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export async function backfillCorrectionKinds(
|
|
43
|
+
options: BackfillOptions = {}
|
|
44
|
+
): Promise<BackfillResult> {
|
|
45
|
+
const label = options.label ?? labelWithInference;
|
|
46
|
+
const replacements = new Map<string, Turn>();
|
|
47
|
+
let failed = 0;
|
|
48
|
+
for (const turn of correctionsToLabel(readTurns(), options.relabel)) {
|
|
49
|
+
const kind = await label(turn);
|
|
50
|
+
options.onLabelled?.(turn, kind);
|
|
51
|
+
if (!kind) failed++;
|
|
52
|
+
else replacements.set(turnKey(turn), { ...turn, ...withCorrectionKind(turn, kind) });
|
|
53
|
+
}
|
|
54
|
+
if (replacements.size > 0) replaceTurns(replacements);
|
|
55
|
+
return { labelled: replacements.size, failed };
|
|
56
|
+
}
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The kind of mistake a confirmed correction points at. Rates of each kind
|
|
3
|
+
* before and after an opinion is approved measure whether the opinion works, so
|
|
4
|
+
* the set is closed; a correction that fits none lands in `other` with the
|
|
5
|
+
* model's own name for it, which a later check can promote to a kind.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import type { TurnInput } from "./adaptation-turns";
|
|
9
|
+
|
|
10
|
+
/** Bump on every change to the kinds or the prompt: rates compare only labels of one version. */
|
|
11
|
+
export const CORRECTION_KINDS_VERSION = 2;
|
|
12
|
+
|
|
13
|
+
export function hasCurrentKind<T extends TurnInput>(
|
|
14
|
+
turn: T
|
|
15
|
+
): turn is T & Required<Pick<TurnInput, "correctionKind">> {
|
|
16
|
+
return turn.correctionKind?.version === CORRECTION_KINDS_VERSION;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
const NAMED_KINDS = {
|
|
20
|
+
"missed-the-question":
|
|
21
|
+
"the reply did not answer what the user asked, or buried the answer",
|
|
22
|
+
"unverified-claim":
|
|
23
|
+
"the reply stated a fact or the state of something (tests pass, CI is green, a tool is missing, a fix is merged) that was wrong or never checked",
|
|
24
|
+
"off-instruction":
|
|
25
|
+
"the reply went against an instruction the user had given in an earlier message: it reordered their steps, worked on a different target than the one they named, changed the scope, or put the work somewhere else, such as a separate PR",
|
|
26
|
+
"revisited-decision":
|
|
27
|
+
"the reply proposed or kept something the user had already ruled out or decided against earlier",
|
|
28
|
+
"stopped-short":
|
|
29
|
+
"the reply deferred, paused or handed back work the user wanted done now, or waited for something (a merge, a review, an answer) instead of starting",
|
|
30
|
+
"wrong-format":
|
|
31
|
+
"the reply ignored the shape or length the user asked for, such as a table, a list or one line, or padded the answer",
|
|
32
|
+
"no-verdict":
|
|
33
|
+
"the user wanted a judgement and the reply gave none: it agreed without reasoning, listed options without picking one, or criticised without an alternative",
|
|
34
|
+
"dismissed-user-view":
|
|
35
|
+
"the reply argued past or brushed aside the user's own view or hypothesis",
|
|
36
|
+
"undefined-reference":
|
|
37
|
+
"the reply used a term, name, number or id the user did not know or had not been told; the user says they don't know it, or asks what it is or to list it",
|
|
38
|
+
"skipped-existing":
|
|
39
|
+
"the reply designed or built something without checking what already exists: code, a mechanism, a tool, a convention or prior art",
|
|
40
|
+
"status-quo-bias":
|
|
41
|
+
"the reply treated the current setup as a fixed constraint instead of considering a better design",
|
|
42
|
+
} as const;
|
|
43
|
+
|
|
44
|
+
type NamedKind = keyof typeof NAMED_KINDS;
|
|
45
|
+
export type CorrectionKindName = NamedKind | "other";
|
|
46
|
+
|
|
47
|
+
export const CORRECTION_KINDS = [
|
|
48
|
+
...(Object.keys(NAMED_KINDS) as NamedKind[]),
|
|
49
|
+
"other",
|
|
50
|
+
] as const satisfies readonly CorrectionKindName[];
|
|
51
|
+
|
|
52
|
+
export interface CorrectionKind {
|
|
53
|
+
kind: CorrectionKindName;
|
|
54
|
+
suggested: string;
|
|
55
|
+
description: string;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const KIND_LINES = Object.entries(NAMED_KINDS)
|
|
59
|
+
.map(([kind, meaning]) => `- ${kind}: ${meaning}.`)
|
|
60
|
+
.join("\n");
|
|
61
|
+
|
|
62
|
+
export const CORRECTION_KIND_SYSTEM = `The user corrected an AI assistant's reply. Name the kind of mistake the reply made.
|
|
63
|
+
|
|
64
|
+
Pick the one kind that best explains why the user had to correct the reply:
|
|
65
|
+
${KIND_LINES}
|
|
66
|
+
- other: none of the kinds above fits.
|
|
67
|
+
|
|
68
|
+
Judge the mistake, not the topic: a wrong claim about CI is unverified-claim, and a question about CI the reply never answered is missed-the-question.
|
|
69
|
+
|
|
70
|
+
The user's correction is often a question itself. That alone does not make it missed-the-question: missed-the-question means the reply failed to answer something the user had asked before the reply. When the correction asks a new question, find the mistake that made the user ask it.
|
|
71
|
+
|
|
72
|
+
The first reader's words are a hint, not the answer. It writes "did not answer" or "did not address the question" for many kinds of mistake, so do not pick missed-the-question because of that wording; judge the reply and the message yourself.
|
|
73
|
+
|
|
74
|
+
missed-the-question is the most general kind, so pick it last: only when no more specific kind below explains the mistake.
|
|
75
|
+
|
|
76
|
+
When kinds overlap, prefer the most specific one:
|
|
77
|
+
- the user does not know or understand something the reply referred to ("what is X?", "I don't get", "list them"): undefined-reference, even when they ask about it;
|
|
78
|
+
- the user asked whether something is worth it, right, or should be done, and the reply gave no clear yes or no: no-verdict;
|
|
79
|
+
- the user asked why, and the reply argued for something else instead of giving the reason: missed-the-question, not dismissed-user-view, even when the issue says the reply contradicted the user. dismissed-user-view needs a view or idea the user had put forward that the reply rejected;
|
|
80
|
+
- the user says they meant a different thing than the reply worked on: off-instruction;
|
|
81
|
+
- a correction nearly always tells the assistant what to do instead. That alone is never off-instruction, which needs an instruction from an earlier message that the reply broke;- the user reminds the reply of something agreed or ruled out earlier ("didn't we say", "we discussed"): revisited-decision;
|
|
82
|
+
- the reply put off work the user wanted now, or said it would start only after something else happened (a merge, a review, an answer): stopped-short, even when it also changed the order.
|
|
83
|
+
|
|
84
|
+
Pick a kind only when its description fits the mistake itself. When a kind fits only loosely, answer other.
|
|
85
|
+
|
|
86
|
+
For other, set suggested to a short lowercase kebab-case name for the mistake, and description to one sentence saying what the reply did wrong. For any other kind, set both to an empty string.`;
|
|
87
|
+
|
|
88
|
+
const CORRECTION_KIND_SCHEMA = {
|
|
89
|
+
type: "object",
|
|
90
|
+
properties: {
|
|
91
|
+
kind: { enum: [...CORRECTION_KINDS] },
|
|
92
|
+
suggested: { type: "string" },
|
|
93
|
+
description: { type: "string" },
|
|
94
|
+
},
|
|
95
|
+
required: ["kind", "suggested", "description"],
|
|
96
|
+
additionalProperties: false,
|
|
97
|
+
} as const;
|
|
98
|
+
|
|
99
|
+
export function correctionKindSchema() {
|
|
100
|
+
return CORRECTION_KIND_SCHEMA;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const MESSAGE_MAX = 800;
|
|
104
|
+
|
|
105
|
+
export function correctionKindUser(replyEnd: string, message: string, issue: string) {
|
|
106
|
+
return `The end of the assistant's reply:\n<reply_end>\n${replyEnd}\n</reply_end>\n\nThe user's correction:\n<message>\n${message.slice(0, MESSAGE_MAX)}\n</message>\n\nWhat the reply got wrong, in a first reader's words:\n<issue>\n${issue}\n</issue>`;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export function correctionKindRequest(
|
|
110
|
+
replyEnd: string,
|
|
111
|
+
message: string,
|
|
112
|
+
issue: string,
|
|
113
|
+
sessionId?: string
|
|
114
|
+
) {
|
|
115
|
+
return {
|
|
116
|
+
system: CORRECTION_KIND_SYSTEM,
|
|
117
|
+
user: correctionKindUser(replyEnd, message, issue),
|
|
118
|
+
maxTokens: 200,
|
|
119
|
+
timeout: 90_000,
|
|
120
|
+
jsonSchema: CORRECTION_KIND_SCHEMA,
|
|
121
|
+
caller: "correction-kind",
|
|
122
|
+
sessionId,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function text(value: unknown): string {
|
|
127
|
+
return typeof value === "string" ? value.trim() : "";
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function isCorrectionKind(value: unknown): value is CorrectionKindName {
|
|
131
|
+
return CORRECTION_KINDS.includes(value as CorrectionKindName);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function unlistedKind(kind: unknown, description: string): CorrectionKind | null {
|
|
135
|
+
const suggested = text(kind);
|
|
136
|
+
return suggested ? { kind: "other", suggested, description } : null;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
export function parseCorrectionKind(
|
|
140
|
+
output: string | null | undefined
|
|
141
|
+
): CorrectionKind | null {
|
|
142
|
+
if (!output) return null;
|
|
143
|
+
try {
|
|
144
|
+
const parsed = JSON.parse(output) as Record<string, unknown>;
|
|
145
|
+
const description = text(parsed.description);
|
|
146
|
+
if (!isCorrectionKind(parsed.kind)) return unlistedKind(parsed.kind, description);
|
|
147
|
+
if (parsed.kind !== "other")
|
|
148
|
+
return { kind: parsed.kind, suggested: "", description: "" };
|
|
149
|
+
return { kind: "other", suggested: text(parsed.suggested), description };
|
|
150
|
+
} catch {
|
|
151
|
+
return null;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
export function needsCorrectionKind(turn: TurnInput | null): turn is TurnInput {
|
|
156
|
+
return turn?.confirmed === true;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export function withCorrectionKind(
|
|
160
|
+
turn: TurnInput,
|
|
161
|
+
kind: CorrectionKind | null
|
|
162
|
+
): TurnInput {
|
|
163
|
+
if (!kind) return turn;
|
|
164
|
+
return { ...turn, correctionKind: { ...kind, version: CORRECTION_KINDS_VERSION } };
|
|
165
|
+
}
|