pi-continual-harness 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -7
- package/package.json +5 -5
- package/src/auto-refine.ts +66 -5
- package/src/classify.ts +220 -0
- package/src/config.ts +28 -1
- package/src/corpus.ts +310 -0
- package/src/harness.ts +40 -0
- package/src/index.ts +35 -0
- package/src/inject.ts +51 -19
- package/src/proposer.ts +154 -39
- package/src/refine.ts +34 -5
- package/src/tools.ts +50 -0
- package/src/vmodel.ts +88 -0
package/README.md
CHANGED
|
@@ -63,6 +63,7 @@ Durable I/O (round-trip with pi-reflect):
|
|
|
63
63
|
```
|
|
64
64
|
/harness status # counts + durable layer presence/mtime
|
|
65
65
|
/harness export [path] # layered export (or full snapshot to a path)
|
|
66
|
+
/harness export-corpus [path] # calibration corpus JSONL for pi-reflex (local-only; dedupe pairs + lifecycle)
|
|
66
67
|
/harness import [--prune] [path] # layered import (or single file from a path)
|
|
67
68
|
/harness prune [--decay <days>] # drop items below the importance floor
|
|
68
69
|
/harness keep <id> # nudge importance up (+0.1)
|
|
@@ -144,8 +145,10 @@ Optional config at `~/.pi/agent/harness.json` (missing or malformed → defaults
|
|
|
144
145
|
"dedupe": { "threshold": 0.6, "merge": true },
|
|
145
146
|
"injection": { "enabled": true, "maxTokens": 1500, "maxPerKind": 10, "charsPerToken": 4 },
|
|
146
147
|
"remindRefine": { "enabled": false, "everyTurns": 50 },
|
|
147
|
-
"autoRefine": { "enabled": false, "everyTurns": 100, "commit": false },
|
|
148
|
-
"outcomeImportance": { "enabled": false, "bump": 0.03 }
|
|
148
|
+
"autoRefine": { "enabled": false, "everyTurns": 100, "commit": false, "gate": false },
|
|
149
|
+
"outcomeImportance": { "enabled": false, "bump": 0.03 },
|
|
150
|
+
"classifier": { "model": "typesafe/jev-latest" },
|
|
151
|
+
"virtualBinding": "virtual"
|
|
149
152
|
}
|
|
150
153
|
```
|
|
151
154
|
|
|
@@ -186,7 +189,29 @@ Optional config at `~/.pi/agent/harness.json` (missing or malformed → defaults
|
|
|
186
189
|
(default 100). It is one of the package's opt-in autonomous paths: it reuses
|
|
187
190
|
the exact `/refine` routine (audited `REFINE_ENTRY` tagged `source: "auto"`,
|
|
188
191
|
branch-local, `/tree` rollback) and notifies before firing. `commit: true`
|
|
189
|
-
also flushes durable state on each run.
|
|
192
|
+
also flushes durable state on each run. `gate: true` (needs `classifier.model`)
|
|
193
|
+
asks the classifier first whether the recent trajectory contains a durable,
|
|
194
|
+
reusable correction and skips the refine when it does not — cadence becomes
|
|
195
|
+
signal, not just time (errors fall back to the plain cadence). The steering
|
|
196
|
+
prompt is delivered via the pi ≥ 0.87 actionable `turn_end` boundary (a
|
|
197
|
+
structural `harness.auto-refine-request` entry + `continue: true`), not a
|
|
198
|
+
synthetic user message.
|
|
199
|
+
- **`classifier`** — opt-in classifier model (pi ≥ 0.99) for the cheap gated
|
|
200
|
+
paths: `"provider/id"` or a bare id of a **classifier** catalog entry, e.g.
|
|
201
|
+
`"typesafe/jev-latest"` or a local `llama-cpp` classifier (every chat model
|
|
202
|
+
listed for llama.cpp also appears as a classifier). When set and resolvable
|
|
203
|
+
it enables the classifier-confirmed dedupe (see `dedupe` above) and the
|
|
204
|
+
auto-refine `gate`. When unset or unresolvable, every classifier feature
|
|
205
|
+
degrades to its no-classifier behavior.
|
|
206
|
+
- **`virtualBinding`** — how the per-model isolation key treats pi **virtual
|
|
207
|
+
models** (router extensions, pi ≥ 0.99). Default `"virtual"`: the key is
|
|
208
|
+
the selected model's own `provider/id` — a router starts a blank harness.
|
|
209
|
+
`"physical"`: when the selection is virtual, the key is the *physical*
|
|
210
|
+
model behind the latest successful response on the branch — notes
|
|
211
|
+
accumulated under that physical model keep injecting through the router, and
|
|
212
|
+
switching routed models re-scopes exactly like a manual model switch. Before
|
|
213
|
+
the first response the key is unknown: creates stay orphans and are adopted
|
|
214
|
+
on first contact.
|
|
190
215
|
- **`outcomeImportance`** — opt-in autonomous **promotion** loop (**off by
|
|
191
216
|
default**). When `enabled`, a `turn_end` hook bumps (+`bump`, default 0.03)
|
|
192
217
|
the importance of any active item the agent cited by its `[h_xxxx]` tag in the
|
|
@@ -315,7 +340,7 @@ deltas directly). The propose stage is pluggable via a registry
|
|
|
315
340
|
|
|
316
341
|
The dedupe planner's `similarity` seam (`DedupeOptions.similarity`, re-exported
|
|
317
342
|
from the package entry) is how a companion package upgrades the comparison —
|
|
318
|
-
e.g. embedding cosine similarity, or a local decision engine such as pi-
|
|
343
|
+
e.g. embedding cosine similarity, or a local decision engine such as pi-reflex
|
|
319
344
|
(a local-only project for now). A comparator may return
|
|
320
345
|
`{ score, abstain }` instead of a plain number: **an abstaining pair is
|
|
321
346
|
treated as not duplicates** — both items are kept — so an engine with
|
|
@@ -369,7 +394,7 @@ Then `"my-proposer"` is selectable via `/refine --proposer my-proposer` or
|
|
|
369
394
|
|
|
370
395
|
## Status
|
|
371
396
|
|
|
372
|
-
0.
|
|
397
|
+
0.12.x. Requires **pi ≥ 0.99.0**. Implemented:
|
|
373
398
|
|
|
374
399
|
- Unified harness-state store with branch-local snapshots (`/tree` rollback).
|
|
375
400
|
- Online `/refine` + `harness_mutate` / `harness_list` tools.
|
|
@@ -394,11 +419,28 @@ Then `"my-proposer"` is selectable via `/refine --proposer my-proposer` or
|
|
|
394
419
|
- **Per-model isolation**: every item is bound to a `provider/id` and injected
|
|
395
420
|
only for that model; new items are stamped automatically and orphans adopted
|
|
396
421
|
on first contact. A new model id starts from a blank harness, and the durable
|
|
397
|
-
round-trip preserves the owner tag.
|
|
422
|
+
round-trip preserves the owner tag. With pi virtual models (routers), the
|
|
423
|
+
`"virtualBinding": "physical"` policy keys items by the routed physical
|
|
424
|
+
model instead of the router id, so a router selection does not silently
|
|
425
|
+
reset the harness.
|
|
398
426
|
- **Bounded injection (on by default)**: the supplemental block is
|
|
399
427
|
importance-ordered and capped per kind + by a total token budget, so a growing
|
|
400
428
|
harness never balloons the system prompt. Tunable / opt-out via the `injection`
|
|
401
|
-
config key; the store is never changed by selection.
|
|
429
|
+
config key; the store is never changed by selection. Rendered on **every
|
|
430
|
+
provider request** (`context_with_system`), so a mid-run `harness_mutate`
|
|
431
|
+
shows up on the very next request.
|
|
432
|
+
- **Classifier features (pi ≥ 0.99, opt-in)**: `"classifier": { "model": … }`
|
|
433
|
+
enables (1) classifier-confirmed dedupe — candidate merge pairs confirmed by
|
|
434
|
+
one batched yes/no call (errors fall back to the rule-based plan) — and (2)
|
|
435
|
+
the auto-refine gate `"autoRefine": { "gate": true }`, which skips a cadenced
|
|
436
|
+
refine when the trajectory holds no durable correction.
|
|
437
|
+
- **Structured tool results**: `harness_list` / `harness_mutate` return
|
|
438
|
+
`structuredContent` (with `outputSchema`), so codemode scripts and
|
|
439
|
+
`ctx.executeTool()` callers can consume harness state programmatically.
|
|
440
|
+
- **Boundary delivery for auto-refine**: the steering prompt is persisted as a
|
|
441
|
+
structural entry with `{ entries, continue: true }` at `turn_end` (no
|
|
442
|
+
synthetic user message, no follow-up-scheduling side effects); manual
|
|
443
|
+
`/refine` still steers via a user message.
|
|
402
444
|
|
|
403
445
|
Open extension points (see `docs/ROADMAP.md`):
|
|
404
446
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-continual-harness",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.12.0",
|
|
4
4
|
"description": "Online self-improvement layer for pi. A unified, in-trajectory harness-state store (prompt notes, memory, skill descriptions, sub-agent specs) with a manual /refine that proposes evidence-backed structured CRUD deltas. Composes with pi-reflect (offline) and pi-mem (storage); owns only the online optimizer layer. Inspired by Continual Harness (arXiv 2605.09998) and ACE (arXiv 2510.04618).",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"pi-package",
|
|
@@ -35,13 +35,13 @@
|
|
|
35
35
|
"node": ">=20.0.0"
|
|
36
36
|
},
|
|
37
37
|
"peerDependencies": {
|
|
38
|
-
"@earendil-works/pi-coding-agent": ">=0.
|
|
39
|
-
"@earendil-works/pi-ai": ">=0.
|
|
38
|
+
"@earendil-works/pi-coding-agent": ">=0.99.0",
|
|
39
|
+
"@earendil-works/pi-ai": ">=0.99.0",
|
|
40
40
|
"typebox": ">=1.0.0"
|
|
41
41
|
},
|
|
42
42
|
"devDependencies": {
|
|
43
|
-
"@earendil-works/pi-coding-agent": ">=0.
|
|
44
|
-
"@earendil-works/pi-ai": ">=0.
|
|
43
|
+
"@earendil-works/pi-coding-agent": ">=0.99.0",
|
|
44
|
+
"@earendil-works/pi-ai": ">=0.99.0",
|
|
45
45
|
"typebox": ">=1.0.0",
|
|
46
46
|
"@types/node": "^22.0.0",
|
|
47
47
|
"typescript": "^5.6.0",
|
package/src/auto-refine.ts
CHANGED
|
@@ -9,13 +9,30 @@
|
|
|
9
9
|
// properties: structured evidence-backed deltas, an audited REFINE_ENTRY
|
|
10
10
|
// (tagged source: "auto"), branch-local snapshots, and /tree rollback.
|
|
11
11
|
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
12
|
+
// Two pi ≥ 0.87/0.99 upgrades (0.12.0):
|
|
13
|
+
//
|
|
14
|
+
// - Classifier gate (opt-in via autoRefine.gate + classifier.model): before
|
|
15
|
+
// spending a refine, ask ONE cheap yes/no classifier question over the
|
|
16
|
+
// recent trajectory. "No durable correction" skips the refine entirely —
|
|
17
|
+
// cadence becomes signal, not just time. Soft-fails to the plain cadence on
|
|
18
|
+
// any classifier error (the gate can skip work, never block it).
|
|
19
|
+
//
|
|
20
|
+
// - Boundary delivery: the steering message is no longer sent as a synthetic
|
|
21
|
+
// user message; runRefine returns it undelivered and this handler persists
|
|
22
|
+
// it as a structural custom-message entry with `{ entries, continue: true }`
|
|
23
|
+
// — the actionable turn_end boundary. One guaranteed next provider request,
|
|
24
|
+
// no steering queue and no follow-up scheduling side effects. Falls back to
|
|
25
|
+
// sendUserMessage when the boundary cannot continue (event.context.
|
|
26
|
+
// canContinue false, e.g. after an error turn).
|
|
27
|
+
//
|
|
28
|
+
// It stays visible: it notifies before firing, and the boundary draft (and the
|
|
29
|
+
// gate decision) appear in the transcript. The decision logic is factored into
|
|
14
30
|
// evaluateAutoRefine() so it is unit-testable without a live pi runtime.
|
|
15
31
|
|
|
16
32
|
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
17
33
|
import { DEFAULT_AUTO_EVERY_TURNS, type HarnessConfig, loadConfig } from "./config.js";
|
|
18
|
-
import { runRefine } from "./refine.js";
|
|
34
|
+
import { runRefine, gatherEvidence, DEFAULT_LOOKBACK_TURNS } from "./refine.js";
|
|
35
|
+
import { buildClassify, buildGateRequest, gateDecision } from "./classify.js";
|
|
19
36
|
|
|
20
37
|
// turnIndex of the last auto-refine (or the seeded baseline). -1 = unseen.
|
|
21
38
|
let lastTurn = -1;
|
|
@@ -45,20 +62,64 @@ export function evaluateAutoRefine(config: HarnessConfig, turnIndex: number): bo
|
|
|
45
62
|
return false;
|
|
46
63
|
}
|
|
47
64
|
|
|
65
|
+
/** The boundary draft custom type for auto-refine steering messages. */
|
|
66
|
+
export const AUTO_REFINE_ENTRY = "harness.auto-refine-request";
|
|
67
|
+
|
|
68
|
+
/** Build the boundary draft carrying a steering message. Pure — testable. */
|
|
69
|
+
export function buildRefineDraft(steeringMessage: string): {
|
|
70
|
+
type: "custom_message";
|
|
71
|
+
customType: string;
|
|
72
|
+
content: string;
|
|
73
|
+
display: boolean;
|
|
74
|
+
} {
|
|
75
|
+
return { type: "custom_message", customType: AUTO_REFINE_ENTRY, content: steeringMessage, display: true };
|
|
76
|
+
}
|
|
77
|
+
|
|
48
78
|
/** Subscribe to turn_end and run /refine on the configured cadence. */
|
|
49
79
|
export function registerAutoRefine(pi: ExtensionAPI): void {
|
|
50
80
|
pi.on("turn_end", async (event, ctx) => {
|
|
51
81
|
const config = await loadConfig();
|
|
52
82
|
if (!evaluateAutoRefine(config, event.turnIndex)) return;
|
|
53
83
|
const every = config.autoRefine?.everyTurns ?? DEFAULT_AUTO_EVERY_TURNS;
|
|
84
|
+
|
|
85
|
+
// Opt-in classifier gate: cheap yes/no before spending the refine.
|
|
86
|
+
if (config.autoRefine?.gate) {
|
|
87
|
+
const classify = buildClassify(ctx, config.classifier?.model);
|
|
88
|
+
if (classify) {
|
|
89
|
+
const evidence = gatherEvidence(ctx, DEFAULT_LOOKBACK_TURNS);
|
|
90
|
+
const res = await classify(buildGateRequest(evidence));
|
|
91
|
+
const decision = gateDecision(res);
|
|
92
|
+
if (!decision.proceed) {
|
|
93
|
+
ctx.ui.notify(`Auto-refine: skipped — ${decision.because}.`, "info");
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
ctx.ui.notify(`Auto-refine: gate passed (${decision.because}).`, "info");
|
|
97
|
+
}
|
|
98
|
+
// No resolvable classifier → plain cadence behavior (documented soft-fail).
|
|
99
|
+
}
|
|
100
|
+
|
|
54
101
|
ctx.ui.notify(`Auto-refine: running /refine (every ${every} turns, opt-in).`, "info");
|
|
55
102
|
try {
|
|
56
|
-
await runRefine(
|
|
103
|
+
const result = await runRefine(
|
|
57
104
|
pi,
|
|
58
105
|
ctx,
|
|
59
|
-
{
|
|
106
|
+
{
|
|
107
|
+
commit: config.autoRefine?.commit ?? false,
|
|
108
|
+
...(config.proposer ? { proposer: config.proposer } : {}),
|
|
109
|
+
delivery: "boundary",
|
|
110
|
+
},
|
|
60
111
|
"auto",
|
|
61
112
|
);
|
|
113
|
+
// Boundary delivery: persist the steering message as a structural entry
|
|
114
|
+
// and guarantee one next provider request. Only when the boundary CAN
|
|
115
|
+
// continue — otherwise fall back to the legacy steering send.
|
|
116
|
+
if (result.steeringMessage) {
|
|
117
|
+
if (event.context?.canContinue === false) {
|
|
118
|
+
pi.sendUserMessage(result.steeringMessage);
|
|
119
|
+
return;
|
|
120
|
+
}
|
|
121
|
+
return { entries: [...event.entries, buildRefineDraft(result.steeringMessage)], continue: true };
|
|
122
|
+
}
|
|
62
123
|
} catch (err) {
|
|
63
124
|
ctx.ui.notify(`Auto-refine failed: ${(err as Error).message}`, "error");
|
|
64
125
|
}
|
package/src/classify.ts
ADDED
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
// Classifier adapter (pi ≥ 0.99). Wraps ctx.modelRegistry.classify() — cheap
|
|
2
|
+
// typed yes/no or score questions answered from next-token label probabilities
|
|
3
|
+
// (local llama.cpp) or a hosted Jev model — behind two package seams:
|
|
4
|
+
//
|
|
5
|
+
// 1. autoRefine gate (auto-refine.ts): when the cadence elapses, ask ONE bool
|
|
6
|
+
// question over the recent trajectory ("did it contain a durable, reusable
|
|
7
|
+
// correction?") and skip the refine when the answer is no. Soft-fails to the
|
|
8
|
+
// old cadence behavior on any classifier error, so a flaky classifier never
|
|
9
|
+
// blocks refinement.
|
|
10
|
+
// 2. dedupe confirmation (proposer.ts): a precision filter on candidate merge
|
|
11
|
+
// pairs — see classifyConfirmPairs().
|
|
12
|
+
//
|
|
13
|
+
// The pi-ai classifier shapes (ClassifierContext/ClassifierQuestion/…) are kept
|
|
14
|
+
// OUT of the public proposer interface: ClassifyFn below is a minimal structural
|
|
15
|
+
// contract so proposers stay testable without pi runtime types.
|
|
16
|
+
|
|
17
|
+
import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
18
|
+
|
|
19
|
+
/** One yes/no question in a ClassifyFn request. */
|
|
20
|
+
export interface ClassifyBoolQuestion {
|
|
21
|
+
instructions: string;
|
|
22
|
+
/** Criteria text for the yes answer. */
|
|
23
|
+
trueCriteria: string;
|
|
24
|
+
/** Criteria text for the no answer. */
|
|
25
|
+
falseCriteria: string;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** A parsed answer per question key. */
|
|
29
|
+
export interface ClassifyBoolAnswer {
|
|
30
|
+
/** The classifier's decision. */
|
|
31
|
+
value: boolean;
|
|
32
|
+
/** [0,1] confidence when the engine reports one. */
|
|
33
|
+
confidence?: number;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Result of a ClassifyFn call: one answer per question key, or an error. */
|
|
37
|
+
export interface ClassifyBoolResult {
|
|
38
|
+
ok: boolean;
|
|
39
|
+
answers: Record<string, ClassifyBoolAnswer>;
|
|
40
|
+
/** Resolved classifier label ("provider/id") for telemetry. */
|
|
41
|
+
model?: string;
|
|
42
|
+
/** Token usage when the engine reports it (for the audit trail). */
|
|
43
|
+
usage?: { input: number; output: number };
|
|
44
|
+
error?: string;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Batched yes/no classification seam injected into ProposeInput when a
|
|
49
|
+
* classifier model is configured AND resolvable. Structural on purpose —
|
|
50
|
+
* the adapter below is the only place that knows pi-ai's shapes.
|
|
51
|
+
*/
|
|
52
|
+
export type ClassifyFn = (input: {
|
|
53
|
+
state: Record<string, unknown>;
|
|
54
|
+
questions: Record<string, ClassifyBoolQuestion>;
|
|
55
|
+
}) => Promise<ClassifyBoolResult>;
|
|
56
|
+
|
|
57
|
+
/** Structural slice of ctx.modelRegistry used here. */
|
|
58
|
+
interface RegistryLike {
|
|
59
|
+
classify?: (model: unknown, context: unknown, options?: unknown) => Promise<ClassifyRawResult>;
|
|
60
|
+
findOfType?: (type: "classifier", provider: string, id: string) => unknown | undefined;
|
|
61
|
+
getModelsOfType?: (type: "classifier") => readonly unknown[];
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Raw pi-ai ClassifierResult slice (never rejects, but stopReason may be error). */
|
|
65
|
+
interface ClassifyRawResult {
|
|
66
|
+
answers?: Record<string, { type?: string; choice?: unknown; score?: number; probability?: number; confidence?: number }>;
|
|
67
|
+
stopReason?: string;
|
|
68
|
+
errorMessage?: string;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Raw classifier-model slice (Model<ClassifierApi>). */
|
|
72
|
+
interface ClassifierModelLike {
|
|
73
|
+
provider?: string;
|
|
74
|
+
id?: string;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Resolve a classifier model from a configured id ("provider/id" or bare id).
|
|
79
|
+
* Returns undefined when unconfigured or unresolvable — every feature keyed on
|
|
80
|
+
* it then degrades to its no-classifier behavior.
|
|
81
|
+
*/
|
|
82
|
+
export function resolveClassifierModel(
|
|
83
|
+
registry: RegistryLike | undefined,
|
|
84
|
+
configured?: string,
|
|
85
|
+
): ClassifierModelLike | undefined {
|
|
86
|
+
if (!configured || !registry) return undefined;
|
|
87
|
+
const slash = configured.indexOf("/");
|
|
88
|
+
if (slash > 0 && typeof registry.findOfType === "function") {
|
|
89
|
+
const byPair = registry.findOfType("classifier", configured.slice(0, slash), configured.slice(slash + 1));
|
|
90
|
+
if (byPair) return byPair as ClassifierModelLike;
|
|
91
|
+
}
|
|
92
|
+
const all =
|
|
93
|
+
typeof registry.getModelsOfType === "function"
|
|
94
|
+
? [...registry.getModelsOfType("classifier")]
|
|
95
|
+
: [];
|
|
96
|
+
return (all.find((m) => {
|
|
97
|
+
const cm = m as ClassifierModelLike;
|
|
98
|
+
return cm.id === configured || `${cm.provider}/${cm.id}` === configured;
|
|
99
|
+
}) ?? undefined) as ClassifierModelLike | undefined;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Build the ClassifyFn adapter over a resolved classifier model. */
|
|
103
|
+
export function makeClassifyFn(
|
|
104
|
+
registry: RegistryLike,
|
|
105
|
+
model: ClassifierModelLike,
|
|
106
|
+
signal?: AbortSignal,
|
|
107
|
+
): ClassifyFn {
|
|
108
|
+
const label = `${model.provider}/${model.id}`;
|
|
109
|
+
return async ({ state, questions }) => {
|
|
110
|
+
if (typeof registry.classify !== "function") {
|
|
111
|
+
return { ok: false, answers: {}, model: label, error: "modelRegistry.classify unavailable" };
|
|
112
|
+
}
|
|
113
|
+
// Map our minimal bool contract onto pi-ai's ClassifierBoolQuestion.
|
|
114
|
+
const piQuestions: Record<string, unknown> = {};
|
|
115
|
+
for (const [key, q] of Object.entries(questions)) {
|
|
116
|
+
piQuestions[key] = {
|
|
117
|
+
type: "bool",
|
|
118
|
+
instructions: q.instructions,
|
|
119
|
+
criteria: { true: q.trueCriteria, false: q.falseCriteria },
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
let raw: ClassifyRawResult;
|
|
123
|
+
try {
|
|
124
|
+
raw = (await registry.classify(model, { state, questions: piQuestions }, signal ? { signal } : undefined)) as ClassifyRawResult;
|
|
125
|
+
} catch (err) {
|
|
126
|
+
// classify "never rejects", but stay defensive around transport edges.
|
|
127
|
+
return { ok: false, answers: {}, model: label, error: (err as Error).message };
|
|
128
|
+
}
|
|
129
|
+
const usage =
|
|
130
|
+
typeof (raw as { usage?: { input?: number; output?: number } }).usage?.input === "number" &&
|
|
131
|
+
typeof (raw as { usage?: { input?: number; output?: number } }).usage?.output === "number"
|
|
132
|
+
? {
|
|
133
|
+
input: (raw as { usage?: { input: number; output: number } }).usage!.input,
|
|
134
|
+
output: (raw as { usage?: { input: number; output: number } }).usage!.output,
|
|
135
|
+
}
|
|
136
|
+
: undefined;
|
|
137
|
+
if (raw.stopReason && raw.stopReason !== "stop") {
|
|
138
|
+
return {
|
|
139
|
+
ok: false,
|
|
140
|
+
answers: {},
|
|
141
|
+
model: label,
|
|
142
|
+
...(usage ? { usage } : {}),
|
|
143
|
+
error: raw.errorMessage ?? `stopReason ${raw.stopReason}`,
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
const answers: Record<string, ClassifyBoolAnswer> = {};
|
|
147
|
+
for (const [key, a] of Object.entries(raw.answers ?? {})) {
|
|
148
|
+
let value: boolean | undefined;
|
|
149
|
+
let confidence: number | undefined;
|
|
150
|
+
if (a?.type === "bool") {
|
|
151
|
+
// Bool answers carry the probability of TRUE.
|
|
152
|
+
value = (a.probability ?? 0) >= 0.5;
|
|
153
|
+
confidence = a.probability;
|
|
154
|
+
} else if (a?.type === "choice") {
|
|
155
|
+
value = a.choice === true || a.choice === "true";
|
|
156
|
+
confidence = a.confidence;
|
|
157
|
+
} else if (a?.type === "score") {
|
|
158
|
+
value = (a.score ?? 0) >= 0.5;
|
|
159
|
+
confidence = a.confidence;
|
|
160
|
+
}
|
|
161
|
+
if (value !== undefined) {
|
|
162
|
+
answers[key] = { value, ...(typeof confidence === "number" ? { confidence } : {}) };
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
return { ok: true, answers, model: label, ...(usage ? { usage } : {}) };
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* Resolve the ClassifyFn for a session from config + ctx, or undefined.
|
|
171
|
+
* The single construction point shared by refine (ProposeInput.classify) and
|
|
172
|
+
* the auto-refine gate.
|
|
173
|
+
*/
|
|
174
|
+
export function buildClassify(
|
|
175
|
+
ctx: ExtensionContext,
|
|
176
|
+
configuredModel?: string,
|
|
177
|
+
): ClassifyFn | undefined {
|
|
178
|
+
const registry = ctx.modelRegistry as RegistryLike | undefined;
|
|
179
|
+
const model = resolveClassifierModel(registry, configuredModel);
|
|
180
|
+
if (!registry || !model) return undefined;
|
|
181
|
+
return makeClassifyFn(registry, model, ctx.signal);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* The auto-refine gate question: did the recent trajectory contain a durable,
|
|
186
|
+
* reusable correction worth a /refine pass? Pure builder — unit-testable.
|
|
187
|
+
* Keeps the state payload bounded (the caller already bounds evidence bytes).
|
|
188
|
+
*/
|
|
189
|
+
export function buildGateRequest(evidence: string): {
|
|
190
|
+
state: Record<string, unknown>;
|
|
191
|
+
questions: Record<string, ClassifyBoolQuestion>;
|
|
192
|
+
} {
|
|
193
|
+
return {
|
|
194
|
+
state: { trajectory: evidence },
|
|
195
|
+
questions: {
|
|
196
|
+
gate: {
|
|
197
|
+
instructions:
|
|
198
|
+
"You are gating an online self-improvement pass for a coding agent. Does the recent trajectory below contain a durable, reusable correction — a mistake fixed, a preference learned, a convention discovered, or a fact worth remembering across sessions — that is NOT already routine work?",
|
|
199
|
+
trueCriteria:
|
|
200
|
+
"The trajectory contains at least one durable, reusable lesson or correction worth persisting into the agent's harness state.",
|
|
201
|
+
falseCriteria:
|
|
202
|
+
"The trajectory is routine work (ordinary edits, reads, test runs) with no reusable correction worth persisting.",
|
|
203
|
+
},
|
|
204
|
+
},
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* Gate decision from a classify result. Errors and missing answers fall back to
|
|
210
|
+
* `true` (proceed) so a flaky classifier never disables refinement — the gate
|
|
211
|
+
* can only SKIP work, never block it.
|
|
212
|
+
*/
|
|
213
|
+
export function gateDecision(result: ClassifyBoolResult): { proceed: boolean; because: string } {
|
|
214
|
+
if (!result.ok) return { proceed: true, because: `classifier error (${result.error ?? "unknown"}); falling back to cadence` };
|
|
215
|
+
const a = result.answers.gate;
|
|
216
|
+
if (!a) return { proceed: true, because: "no gate answer; falling back to cadence" };
|
|
217
|
+
return a.value
|
|
218
|
+
? { proceed: true, because: `classifier: yes (confidence ${a.confidence ?? "?"})` }
|
|
219
|
+
: { proceed: false, because: `classifier: no durable correction (confidence ${a.confidence ?? "?"})` };
|
|
220
|
+
}
|
package/src/config.ts
CHANGED
|
@@ -27,6 +27,12 @@ export const DEFAULT_AUTO_EVERY_TURNS = 100;
|
|
|
27
27
|
/** Default per-reference importance bump for the opt-in outcome loop. */
|
|
28
28
|
export const DEFAULT_REF_BUMP = 0.03;
|
|
29
29
|
|
|
30
|
+
/** How the binding key treats virtual (router) models. "virtual" (default):
|
|
31
|
+
* key = the selected model's own provider/id. "physical": when the selection
|
|
32
|
+
* is virtual, key = the physical model behind the latest successful response
|
|
33
|
+
* on the branch (see src/vmodel.ts). */
|
|
34
|
+
export type VirtualBinding = "virtual" | "physical";
|
|
35
|
+
|
|
30
36
|
/** Resolved dedupe policy (merge-capable, 0.10.0). Populated by loadConfig. */
|
|
31
37
|
export interface NormalizedDedupe {
|
|
32
38
|
/** Token-overlap threshold in (0,1]; pairs at/above it merge. Default 0.6. */
|
|
@@ -51,7 +57,21 @@ export interface HarnessConfig {
|
|
|
51
57
|
enabled?: boolean;
|
|
52
58
|
everyTurns?: number;
|
|
53
59
|
commit?: boolean;
|
|
60
|
+
/** Opt-in classifier gate (pi ≥ 0.99, needs `classifier.model`): when the
|
|
61
|
+
* cadence elapses, ask a cheap yes/no classifier whether the recent
|
|
62
|
+
* trajectory contains a durable correction; skip the refine when it says
|
|
63
|
+
* no. Soft-fails to the plain cadence on classifier errors. Default false. */
|
|
64
|
+
gate?: boolean;
|
|
65
|
+
};
|
|
66
|
+
/** Classifier model for the cheap gated paths (pi ≥ 0.99): "provider/id" or
|
|
67
|
+
* a bare id of a classifier catalog entry (e.g. "typesafe/jev-latest" or a
|
|
68
|
+
* local llama.cpp classifier). When unset or unresolvable, every classifier
|
|
69
|
+
* feature degrades to its no-classifier behavior. */
|
|
70
|
+
classifier?: {
|
|
71
|
+
model?: string | undefined;
|
|
54
72
|
};
|
|
73
|
+
/** Virtual-model binding policy (pi ≥ 0.99). Default "virtual". */
|
|
74
|
+
virtualBinding?: VirtualBinding;
|
|
55
75
|
/** Delta proposer name (see proposer.ts registry). Defaults to "steering". */
|
|
56
76
|
proposer?: string;
|
|
57
77
|
/** Opt-in turn_end outcome loop: promote importance of items the agent
|
|
@@ -72,7 +92,9 @@ export const DEFAULT_CONFIG: HarnessConfig = {
|
|
|
72
92
|
durableScope: "global",
|
|
73
93
|
autoImport: false,
|
|
74
94
|
remindRefine: { enabled: false, everyTurns: DEFAULT_EVERY_TURNS },
|
|
75
|
-
autoRefine: { enabled: false, everyTurns: DEFAULT_AUTO_EVERY_TURNS, commit: false },
|
|
95
|
+
autoRefine: { enabled: false, everyTurns: DEFAULT_AUTO_EVERY_TURNS, commit: false, gate: false },
|
|
96
|
+
classifier: { model: undefined },
|
|
97
|
+
virtualBinding: "virtual",
|
|
76
98
|
proposer: "steering",
|
|
77
99
|
outcomeImportance: { enabled: false, bump: DEFAULT_REF_BUMP },
|
|
78
100
|
// Dedupe (merge-capable): merge ON at the historical 0.6 threshold — a
|
|
@@ -102,7 +124,12 @@ function mergeConfig(over: Partial<HarnessConfig>): HarnessConfig {
|
|
|
102
124
|
enabled: over.autoRefine?.enabled ?? false,
|
|
103
125
|
everyTurns: over.autoRefine?.everyTurns ?? DEFAULT_AUTO_EVERY_TURNS,
|
|
104
126
|
commit: over.autoRefine?.commit ?? false,
|
|
127
|
+
gate: over.autoRefine?.gate === true,
|
|
128
|
+
},
|
|
129
|
+
classifier: {
|
|
130
|
+
model: typeof over.classifier?.model === "string" && over.classifier.model ? over.classifier.model : undefined,
|
|
105
131
|
},
|
|
132
|
+
virtualBinding: over.virtualBinding === "physical" ? "physical" : "virtual",
|
|
106
133
|
proposer: over.proposer ?? "steering",
|
|
107
134
|
outcomeImportance: {
|
|
108
135
|
enabled: over.outcomeImportance?.enabled ?? false,
|