specpi 0.26.0 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +80 -0
- package/README.md +37 -3
- package/SECURITY_MODEL.md +44 -4
- package/THIRD_PARTY.md +9 -1
- package/extensions/jev-advisor/broker.mjs +277 -0
- package/extensions/jev-advisor/client.mjs +172 -0
- package/extensions/jev-advisor/config.mjs +270 -0
- package/extensions/jev-advisor/consent.mjs +133 -0
- package/extensions/jev-advisor/gate.mjs +263 -0
- package/extensions/jev-advisor/index.ts +999 -0
- package/extensions/jev-advisor/key-source.mjs +252 -0
- package/extensions/jev-advisor/layer.mjs +169 -0
- package/extensions/jev-advisor/ledger.mjs +138 -0
- package/extensions/jev-advisor/questions/capabilities.mjs +124 -0
- package/extensions/jev-advisor/questions/compaction.mjs +153 -0
- package/extensions/jev-advisor/questions/gap.mjs +140 -0
- package/extensions/jev-advisor/questions/guard.mjs +168 -0
- package/extensions/jev-advisor/questions/progress.mjs +195 -0
- package/extensions/jev-advisor/questions/retention.mjs +188 -0
- package/extensions/jev-advisor/questions/sources.mjs +91 -0
- package/extensions/jev-advisor/questions/untrusted.mjs +69 -0
- package/extensions/jev-advisor/risk.mjs +442 -0
- package/extensions/jev-advisor/sanitize.mjs +0 -0
- package/extensions/jev-advisor/usage.mjs +92 -0
- package/extensions/tool-wishlist/authoring-tools.mjs +42 -0
- package/extensions/tool-wishlist/index.ts +11 -0
- package/extensions/workflow-controls/capabilities.mjs +26 -0
- package/extensions/workflow-controls/index.ts +2 -2
- package/package.json +1 -1
- package/scripts/packages.mjs +56 -0
- package/scripts/specpi.mjs +73 -4
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
// System 1b: guide compaction, which is the one moment the prompt cache is discarded anyway.
|
|
2
|
+
//
|
|
3
|
+
// Pi's `findCutPoint` is documented as "walk backwards from newest, accumulating estimated message
|
|
4
|
+
// sizes, stop when we've accumulated >= keepRecentTokens". It is a token ruler: it cannot tell the
|
|
5
|
+
// load-bearing finding from six dead-end greps, and it discards whichever falls on the wrong side
|
|
6
|
+
// of the line. Since the prefix is being rebuilt regardless, improving that choice costs nothing.
|
|
7
|
+
//
|
|
8
|
+
// This system never sets the cut itself. It supplies `customInstructions` — an existing documented
|
|
9
|
+
// parameter on the compaction path — so the summariser is told what this session was actually
|
|
10
|
+
// about. The token budget still bounds the result, so bad advice can shape a summary, never blow
|
|
11
|
+
// the budget or drop an entry the preparation meant to keep.
|
|
12
|
+
|
|
13
|
+
import { choice, noul } from "../client.mjs";
|
|
14
|
+
import { choiceValue, nounTrue } from "../gate.mjs";
|
|
15
|
+
import { compact } from "../sanitize.mjs";
|
|
16
|
+
|
|
17
|
+
export const WORK_KINDS = Object.freeze({
|
|
18
|
+
debugging: "Tracking down why something fails",
|
|
19
|
+
building: "Adding or changing a feature",
|
|
20
|
+
refactoring: "Restructuring code without changing behaviour",
|
|
21
|
+
research: "Reading and answering questions about a codebase",
|
|
22
|
+
testing: "Writing or repairing tests",
|
|
23
|
+
ops: "Builds, releases, configuration or tooling",
|
|
24
|
+
review: "Reading a diff and judging it",
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* A digest of what compaction is about to discard: entry kinds and scale, never their text. The
|
|
29
|
+
* summariser still sees the real conversation; this only steers what it keeps.
|
|
30
|
+
*/
|
|
31
|
+
export function buildInput({ preparation, objective }) {
|
|
32
|
+
const messages = preparation?.messagesToSummarize ?? [];
|
|
33
|
+
const kinds = {};
|
|
34
|
+
for (const message of messages) {
|
|
35
|
+
const role = typeof message?.role === "string" ? message.role : "unknown";
|
|
36
|
+
kinds[role] = (kinds[role] ?? 0) + 1;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const files = preparation?.fileOps ?? {};
|
|
40
|
+
|
|
41
|
+
return {
|
|
42
|
+
objective: compact(objective ?? "", 180),
|
|
43
|
+
discarding: messages.length,
|
|
44
|
+
roles: kinds,
|
|
45
|
+
tokensBefore: preparation?.tokensBefore ?? 0,
|
|
46
|
+
splitTurn: preparation?.isSplitTurn === true,
|
|
47
|
+
filesRead: [...(files.read ?? [])].slice(0, 12).map((item) => compact(item, 60)),
|
|
48
|
+
filesWritten: [...(files.written ?? []), ...(files.edited ?? [])].slice(0, 12).map((item) => compact(item, 60)),
|
|
49
|
+
hadPreviousSummary: typeof preparation?.previousSummary === "string",
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* The same digest for a branch being left behind. `/tree` hands a different preparation shape --
|
|
55
|
+
* session entries rather than messages, and no token count, because nothing is being cut to fit a
|
|
56
|
+
* budget -- so it gets its own builder rather than a compaction input with three fields quietly
|
|
57
|
+
* reading undefined.
|
|
58
|
+
*/
|
|
59
|
+
export function buildBranchInput({ preparation, objective }) {
|
|
60
|
+
const entries = preparation?.entriesToSummarize ?? [];
|
|
61
|
+
const kinds = {};
|
|
62
|
+
for (const entry of entries) {
|
|
63
|
+
const kind = typeof entry?.type === "string" ? entry.type : "unknown";
|
|
64
|
+
kinds[kind] = (kinds[kind] ?? 0) + 1;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
return {
|
|
68
|
+
objective: compact(objective ?? "", 180),
|
|
69
|
+
abandoning: entries.length,
|
|
70
|
+
kinds,
|
|
71
|
+
wantsSummary: preparation?.userWantsSummary === true,
|
|
72
|
+
// Navigating to an ancestor is backing out of a line of work; navigating elsewhere is
|
|
73
|
+
// moving between siblings. The distinction is most of what a label has to capture.
|
|
74
|
+
toAncestor: preparation?.targetId === preparation?.commonAncestorId,
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Short, navigational, and a fixed enum so no model-written text reaches the session file. `/tree`
|
|
80
|
+
* can filter to labelled entries, so a branch that says what it was is the difference between a
|
|
81
|
+
* navigable tree and a list of timestamps.
|
|
82
|
+
*/
|
|
83
|
+
export const BRANCH_LABELS = Object.freeze({
|
|
84
|
+
"dead end": "The branch was abandoned because the approach did not work",
|
|
85
|
+
"alternative tried": "A different approach to the same goal, set aside for another",
|
|
86
|
+
"work completed": "The branch finished what it set out to do",
|
|
87
|
+
research: "The branch was reading and answering questions, not changing anything",
|
|
88
|
+
reverted: "The branch's changes were undone",
|
|
89
|
+
interrupted: "The branch stopped part-way for an unrelated reason",
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
export function questions({ branch = false } = {}) {
|
|
93
|
+
return {
|
|
94
|
+
...(branch ? { branch_label: choice("What was this abandoned branch?", BRANCH_LABELS) } : {}),
|
|
95
|
+
work_kind: choice("What kind of work has this session mostly been doing?", WORK_KINDS),
|
|
96
|
+
unresolved_thread: noul("There is an unfinished investigation whose findings must survive compaction"),
|
|
97
|
+
discarded_span_was_dead_ends: noul(
|
|
98
|
+
"The work being discarded was mostly abandoned attempts that led nowhere useful",
|
|
99
|
+
),
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const FOCUS = Object.freeze({
|
|
104
|
+
debugging: "the symptom, what has been ruled out, and the current hypothesis",
|
|
105
|
+
building: "what has been implemented so far and what remains",
|
|
106
|
+
refactoring: "the invariants being preserved and which call sites have been updated",
|
|
107
|
+
research: "the questions answered so far, with the files each answer came from",
|
|
108
|
+
testing: "which tests exist, which fail, and why",
|
|
109
|
+
ops: "the commands run, their outcomes, and the current configuration state",
|
|
110
|
+
review: "the findings raised so far and their severity",
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Build `customInstructions` from gated answers only. With nothing gated this returns undefined and
|
|
115
|
+
* Pi's own default prompt is used unchanged.
|
|
116
|
+
*/
|
|
117
|
+
/**
|
|
118
|
+
* The label for a branch summary entry, or undefined when the answer is ungated. Separate from
|
|
119
|
+
* `decide` because the compaction hook has no label to set and would carry a dead field.
|
|
120
|
+
*/
|
|
121
|
+
export function label(answers) {
|
|
122
|
+
const value = choiceValue(answers?.branch_label, "compaction");
|
|
123
|
+
|
|
124
|
+
return value && Object.hasOwn(BRANCH_LABELS, value) ? value : undefined;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export function decide(answers) {
|
|
128
|
+
const kind = choiceValue(answers?.work_kind, "compaction");
|
|
129
|
+
const unresolved = nounTrue(answers?.unresolved_thread, "compaction");
|
|
130
|
+
const deadEnds = nounTrue(answers?.discarded_span_was_dead_ends, "compaction");
|
|
131
|
+
const parts = [];
|
|
132
|
+
if (kind && FOCUS[kind]) {
|
|
133
|
+
parts.push(`This session has mainly been ${kind}. Prioritise ${FOCUS[kind]}.`);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
if (unresolved) {
|
|
137
|
+
parts.push(
|
|
138
|
+
"An investigation is still open. Preserve its findings and the current hypothesis in full, even at the cost of earlier detail.",
|
|
139
|
+
);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
if (deadEnds) {
|
|
143
|
+
parts.push(
|
|
144
|
+
"Most of the discarded work was abandoned attempts. Record what was ruled out in one line each rather than recounting them, so the same paths are not retried.",
|
|
145
|
+
);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
if (parts.length === 0) {
|
|
149
|
+
return { customInstructions: undefined, deadEnds, unresolved };
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
return { customInstructions: parts.join(" "), deadEnds, unresolved };
|
|
153
|
+
}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
// System 2: triage a capability-gap report before it is written.
|
|
2
|
+
//
|
|
3
|
+
// This fixes two real defects in the improvement loop rather than saving tokens.
|
|
4
|
+
//
|
|
5
|
+
// Fragmentation: `aggregateEvents` groups by exact `canonicalKey` string, with manual alias
|
|
6
|
+
// decisions as the only correction, and qualification needs `priority >= blocked || occurrences
|
|
7
|
+
// >= 2`. A genuinely recurring problem that the model names differently each time fragments into
|
|
8
|
+
// singletons and can stay permanently unqualified — it never reaches the human at
|
|
9
|
+
// /harness-improvement. Lexical matching cannot fix that; semantic clustering can.
|
|
10
|
+
//
|
|
11
|
+
// Self-reported severity: `priority` sums IMPACT_WEIGHT[event.impact], and `impact` is reported by
|
|
12
|
+
// the model about its own gap. The ranking that decides what a human sees rests on the least
|
|
13
|
+
// reliable field in the record. An independent score is recorded *alongside* it, never over it, so
|
|
14
|
+
// the human still sees what the model claimed.
|
|
15
|
+
//
|
|
16
|
+
// Authority is unchanged. Only an exact human selection through /harness-improvement authorizes a
|
|
17
|
+
// wishlist-sourced change, and nothing here writes a decision.
|
|
18
|
+
|
|
19
|
+
import { choice, noul, score } from "../client.mjs";
|
|
20
|
+
import { choiceValue, nounTrue, scoreLevel } from "../gate.mjs";
|
|
21
|
+
import { compact } from "../sanitize.mjs";
|
|
22
|
+
|
|
23
|
+
export const IMPACT_LEVELS = Object.freeze([
|
|
24
|
+
"Minor: a small inconvenience with an easy workaround",
|
|
25
|
+
"Moderate: real friction that cost time or forced a detour",
|
|
26
|
+
"Blocked: the task could not be completed as asked",
|
|
27
|
+
]);
|
|
28
|
+
|
|
29
|
+
export const FIX_KINDS = Object.freeze({
|
|
30
|
+
tool: "A new or changed tool would solve it",
|
|
31
|
+
skill: "A skill or documented procedure would solve it",
|
|
32
|
+
prompt: "Different instructions would solve it",
|
|
33
|
+
config: "A configuration or setting change would solve it",
|
|
34
|
+
bug: "Something is broken and should be repaired",
|
|
35
|
+
unknown: "Not clear from what was observed",
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
export const NEW_CLUSTER = "__new__";
|
|
39
|
+
|
|
40
|
+
/** The Choice ceiling is 255; above the cap, rank by similarity first and choose among the top N. */
|
|
41
|
+
export const MAX_CLUSTER_OPTIONS = 200;
|
|
42
|
+
|
|
43
|
+
function similarity(a, b) {
|
|
44
|
+
const left = new Set(
|
|
45
|
+
String(a)
|
|
46
|
+
.toLowerCase()
|
|
47
|
+
.split(/[^a-z0-9]+/u)
|
|
48
|
+
.filter(Boolean),
|
|
49
|
+
);
|
|
50
|
+
const right = new Set(
|
|
51
|
+
String(b)
|
|
52
|
+
.toLowerCase()
|
|
53
|
+
.split(/[^a-z0-9]+/u)
|
|
54
|
+
.filter(Boolean),
|
|
55
|
+
);
|
|
56
|
+
if (left.size === 0 || right.size === 0) {
|
|
57
|
+
return 0;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
let shared = 0;
|
|
61
|
+
for (const token of left) {
|
|
62
|
+
if (right.has(token)) {
|
|
63
|
+
shared += 1;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
return shared / Math.max(left.size, right.size);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* TypeSafe's own two-stage pattern for high cardinality: score candidates cheaply, then make one
|
|
72
|
+
* explicit choice among the survivors. Below the cap every key is offered.
|
|
73
|
+
*/
|
|
74
|
+
export function clusterOptions(existing, gap) {
|
|
75
|
+
const keys = [...new Set(existing.map((item) => item.canonicalKey).filter(Boolean))];
|
|
76
|
+
const probe = `${gap?.capability ?? ""} ${gap?.scenario ?? ""}`;
|
|
77
|
+
const ranked =
|
|
78
|
+
keys.length <= MAX_CLUSTER_OPTIONS
|
|
79
|
+
? keys
|
|
80
|
+
: keys
|
|
81
|
+
.map((key) => ({ key, rank: similarity(key, probe) }))
|
|
82
|
+
.sort((a, b) => b.rank - a.rank)
|
|
83
|
+
.slice(0, MAX_CLUSTER_OPTIONS)
|
|
84
|
+
.map((item) => item.key);
|
|
85
|
+
|
|
86
|
+
const criteria = { [NEW_CLUSTER]: "This is a distinct problem not already on the list" };
|
|
87
|
+
for (const key of ranked) {
|
|
88
|
+
const match = existing.find((item) => item.canonicalKey === key);
|
|
89
|
+
criteria[key] = compact(match?.title ?? key, 90);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
return criteria;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export function buildInput({ gap, existing = [] }) {
|
|
96
|
+
return {
|
|
97
|
+
capability: compact(gap?.capability ?? "", 120),
|
|
98
|
+
scenario: compact(gap?.scenario ?? "", 200),
|
|
99
|
+
limitation: compact(gap?.limitation ?? "", 160),
|
|
100
|
+
workaround: compact(gap?.workaround ?? "", 160),
|
|
101
|
+
claimedImpact: compact(gap?.impact ?? "", 20),
|
|
102
|
+
knownProblems: existing.slice(0, 8).map((item) => compact(item.title ?? item.canonicalKey, 60)),
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export function questions({ gap, existing = [] }) {
|
|
107
|
+
return {
|
|
108
|
+
cluster: choice(
|
|
109
|
+
"Which known problem is this the same underlying problem as, if any?",
|
|
110
|
+
clusterOptions(existing, gap),
|
|
111
|
+
),
|
|
112
|
+
independent_impact: score("How badly did this actually obstruct the task?", IMPACT_LEVELS),
|
|
113
|
+
suggested_fix: choice("What kind of change would address this?", FIX_KINDS),
|
|
114
|
+
contains_secret_or_path: noul(
|
|
115
|
+
"This report contains a credential, an absolute filesystem path, or other machine-specific detail",
|
|
116
|
+
),
|
|
117
|
+
is_transient_or_user_error: noul(
|
|
118
|
+
"This was a one-off failure or a mistake in how the task was asked, not a reusable gap in the harness",
|
|
119
|
+
),
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const IMPACT_NAMES = Object.freeze(["minor", "moderate", "blocked"]);
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Advice, in the shape the caller applies it. `blockForSanitization` is the only field that stops
|
|
127
|
+
* a write, and it stops it to ask the model to rewrite its own text — never to discard the report.
|
|
128
|
+
*/
|
|
129
|
+
export function decide(answers) {
|
|
130
|
+
const cluster = choiceValue(answers?.cluster, "gap");
|
|
131
|
+
const level = scoreLevel(answers?.independent_impact, "gap");
|
|
132
|
+
|
|
133
|
+
return {
|
|
134
|
+
canonicalKey: cluster && cluster !== NEW_CLUSTER ? cluster : undefined,
|
|
135
|
+
independentImpact: level === undefined ? undefined : IMPACT_NAMES[level],
|
|
136
|
+
suggestedFix: choiceValue(answers?.suggested_fix, "gap"),
|
|
137
|
+
blockForSanitization: nounTrue(answers?.contains_secret_or_path, "gap"),
|
|
138
|
+
transient: nounTrue(answers?.is_transient_or_user_error, "gap"),
|
|
139
|
+
};
|
|
140
|
+
}
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
// System 8: score a shell or file call that local rules could not settle, before it runs.
|
|
2
|
+
//
|
|
3
|
+
// This is the native command guard. It replaces the pinned `specpi-jev-guard` package, and the
|
|
4
|
+
// reason it exists rather than that package being configured is that the package's shape kept
|
|
5
|
+
// producing the same class of defect: its configuration was a global file with no session scope, so
|
|
6
|
+
// there was no such thing as enabling it for one session; it read its key from the environment only,
|
|
7
|
+
// so a credential `/login` had stored was invisible to it; and it was fail-closed, so an outage or a
|
|
8
|
+
// missing key turned every shell call in the session into a refusal.
|
|
9
|
+
//
|
|
10
|
+
// Native, all three go away. The switch is `systems.guard` like every other system, so it is
|
|
11
|
+
// session-scoped, budgeted, reported in `/jev status` and toggled by `/jev enable guard`. The key is
|
|
12
|
+
// whatever `key-source.mjs` resolves, which includes Pi's own credential store. And the failure
|
|
13
|
+
// posture is inverted, deliberately:
|
|
14
|
+
//
|
|
15
|
+
// FAIL OPEN. Jev unreachable, unconfident, out of budget, or without a key means the call goes to
|
|
16
|
+
// @gotgenes/pi-permission-system exactly as it did before this layer existed.
|
|
17
|
+
//
|
|
18
|
+
// That is the rule the rest of this extension already follows -- "nothing here is ever the reason a
|
|
19
|
+
// tool is blocked" -- and applying it here is not a weakening. The guard sits *in front of* the
|
|
20
|
+
// permission system, which still decides every call it defers. Fail-closed meant an outage stopped
|
|
21
|
+
// work; fail-open means an outage returns policy to the component that owned it anyway. The guard
|
|
22
|
+
// adds blocking for confident, dangerous verdicts, and adds nothing else.
|
|
23
|
+
//
|
|
24
|
+
// What it sends is bounded by sanitize.mjs like every other system: a redacted digest, never the raw
|
|
25
|
+
// environment, and never more than the shared 1 KB budget.
|
|
26
|
+
|
|
27
|
+
import { score } from "../client.mjs";
|
|
28
|
+
import { THRESHOLDS, scoreLevel } from "../gate.mjs";
|
|
29
|
+
|
|
30
|
+
/** How a decision was reached, for the ledger and for `/jev status`. */
|
|
31
|
+
export const SOURCES = Object.freeze(["local", "jev", "human", "deferred"]);
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* One Score, not a Noul.
|
|
35
|
+
*
|
|
36
|
+
* A Noul returns a bare probability with no confidence field, and this is the one system whose
|
|
37
|
+
* action takes a tool call away from the person. Gating that on a number with no reported confidence
|
|
38
|
+
* is exactly what the calibration work found unreachable elsewhere: without a confidence the gate
|
|
39
|
+
* either fires on coin flips or never fires at all. A Score reports both, so "confident this is
|
|
40
|
+
* destructive" and "unsure" are distinguishable, and only the first can block.
|
|
41
|
+
*/
|
|
42
|
+
export function questions(input) {
|
|
43
|
+
return {
|
|
44
|
+
risk: score(
|
|
45
|
+
"Rate how likely this call is to destroy data, exfiltrate a credential, or hand control " +
|
|
46
|
+
"of the machine to someone else, given what the person asked for. Judge the call as " +
|
|
47
|
+
"written, not the worst command that shares its first word. Routine development work " +
|
|
48
|
+
"-- building, testing, installing declared dependencies, editing source, committing -- " +
|
|
49
|
+
"is not destructive merely because it writes.",
|
|
50
|
+
[
|
|
51
|
+
"harmless: reads, or writes only within the project in the ordinary course of the task",
|
|
52
|
+
"ordinary: changes state the person would expect this task to change",
|
|
53
|
+
"risky: destroys work, rewrites history, or reaches outside the project, but plausibly intended",
|
|
54
|
+
"destructive: irreversible loss, credential exposure, or remote code execution that the request does not call for",
|
|
55
|
+
],
|
|
56
|
+
),
|
|
57
|
+
intended: score("Rate how well this call matches what the person actually asked for.", [
|
|
58
|
+
"unrelated",
|
|
59
|
+
"tangential",
|
|
60
|
+
"plausible",
|
|
61
|
+
"clearly what was asked",
|
|
62
|
+
]),
|
|
63
|
+
...(input?.protected === true
|
|
64
|
+
? {
|
|
65
|
+
credential: score(
|
|
66
|
+
"Rate how likely this write targets a real secret rather than a template, " +
|
|
67
|
+
"example, fixture or documentation file that merely looks like one.",
|
|
68
|
+
["template or example", "unclear", "likely a real secret", "certainly a real secret"],
|
|
69
|
+
),
|
|
70
|
+
}
|
|
71
|
+
: {}),
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Turn the answers into one of three outcomes.
|
|
77
|
+
*
|
|
78
|
+
* `block` needs a confident destructive reading *and* a confident reading that the request does not
|
|
79
|
+
* account for the call. Both, because the single most likely way to be wrong here is a genuinely
|
|
80
|
+
* destructive-looking command that the person asked for in as many words -- `rm -rf node_modules`, a
|
|
81
|
+
* force push to a branch they named. Requiring the intent answer to actively disagree is what keeps
|
|
82
|
+
* those working.
|
|
83
|
+
*
|
|
84
|
+
* "Both" means both, including when the intent answer does not survive its own confidence gate. An
|
|
85
|
+
* earlier version read a missing intent answer as agreement, which mattered far more than it sounds:
|
|
86
|
+
* `gate.mjs`'s own calibration records a score coverage near 0.2, so roughly four answers in five are
|
|
87
|
+
* ungated and a confident risk=3 would have blocked essentially unconditionally -- turning the one
|
|
88
|
+
* stated safeguard into a clause that almost never applied.
|
|
89
|
+
*
|
|
90
|
+
* `ask` is the middle band, and only when there is a human to ask. Without a UI it becomes `defer`,
|
|
91
|
+
* because a question nobody can answer is a block wearing a friendlier word.
|
|
92
|
+
*
|
|
93
|
+
* Everything else defers to the permission system.
|
|
94
|
+
*/
|
|
95
|
+
export function decide(answers, { hasUI = false } = {}) {
|
|
96
|
+
const risk = answers?.risk;
|
|
97
|
+
const intended = answers?.intended;
|
|
98
|
+
if (!risk) {
|
|
99
|
+
return { action: "defer", source: "deferred", reason: "no answer" };
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const level = scoreLevel(risk, "guard");
|
|
103
|
+
const wanted = intended ? scoreLevel(intended, "guard") : undefined;
|
|
104
|
+
|
|
105
|
+
// A confident secret verdict blocks on its own: a write to a real credential file is not made
|
|
106
|
+
// acceptable by having been asked for, and the fixture case is what the question separates.
|
|
107
|
+
const credential = answers?.credential ? scoreLevel(answers.credential, "guard") : undefined;
|
|
108
|
+
if (credential === 3) {
|
|
109
|
+
return { action: "block", source: "jev", reason: "writes a real credential file" };
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
if (level === 3 && wanted !== undefined && wanted <= 1) {
|
|
113
|
+
return { action: "block", source: "jev", reason: "destructive and not what was asked for" };
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
if (level === 3 || level === 2) {
|
|
117
|
+
return hasUI
|
|
118
|
+
? { action: "ask", source: "human", reason: level === 3 ? "destructive" : "risky" }
|
|
119
|
+
: { action: "defer", source: "deferred", reason: "no interface to ask" };
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
return { action: "defer", source: "deferred", reason: "below the bar" };
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** The two answers the guard's confirmation dialog offers, in the order it offers them. */
|
|
126
|
+
export const CHOICES = Object.freeze({ run: "Run it", block: "Block it" });
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Whether a human's answer to that dialog is consent.
|
|
130
|
+
*
|
|
131
|
+
* Only the affirmative is. Escape, a dismissed picker, a host that resolves with nothing and a host
|
|
132
|
+
* that throws all produce something that is not `CHOICES.run`, and every one of them has to mean
|
|
133
|
+
* "not approved" -- reaching this point means the verdict already said the call needs a person's
|
|
134
|
+
* approval, and an unanswerable question resolved as yes is what a confirmation dialog exists to
|
|
135
|
+
* rule out. It lives here rather than in the handler so that it is a rule with a test, not a
|
|
136
|
+
* comparison inside a closure no test imports.
|
|
137
|
+
*/
|
|
138
|
+
export function approved(choice) {
|
|
139
|
+
return choice === CHOICES.run;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** The gate's own numbers, named so `/jev status` and the tests read the same source. */
|
|
143
|
+
export const GATE = Object.freeze({
|
|
144
|
+
scoreConfidence: THRESHOLDS.guard.scoreConfidence,
|
|
145
|
+
boundary: THRESHOLDS.guard.boundary,
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* The state the question is asked against, bounded like every other system's.
|
|
150
|
+
*
|
|
151
|
+
* What the model needs is the call, what the person asked for, and enough recent history to tell a
|
|
152
|
+
* cleanup step from a first move. What it must not receive is the environment, the file's contents,
|
|
153
|
+
* or anything the sanitiser has not seen: `buildState` applies the shared redaction and the 1 KB
|
|
154
|
+
* ceiling, so a long command arrives truncated rather than in full.
|
|
155
|
+
*/
|
|
156
|
+
export function buildInput({ tool, subject, protectedTarget, objective, recent, cwd }) {
|
|
157
|
+
return {
|
|
158
|
+
system: "command-guard",
|
|
159
|
+
tool: String(tool ?? ""),
|
|
160
|
+
call: String(subject ?? ""),
|
|
161
|
+
protectedTarget: protectedTarget === true,
|
|
162
|
+
// Why the call was made matters as much as what it does: the same command is routine in one
|
|
163
|
+
// task and destructive in another, and the intent question has nothing to weigh without it.
|
|
164
|
+
objective: String(objective ?? ""),
|
|
165
|
+
recent: Array.isArray(recent) ? recent.slice(-6).map((item) => `${item.tool}:${item.outcome}`) : [],
|
|
166
|
+
cwd: String(cwd ?? ""),
|
|
167
|
+
};
|
|
168
|
+
}
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
// System 5: notice that a session has stopped making progress, while it can still be helped.
|
|
2
|
+
//
|
|
3
|
+
// This is the only system aimed at turns rather than at input tokens, and turns are where the money
|
|
4
|
+
// is on the hard tiers. Splitting recorded cost three ways: at tier 3 output is 59% of spend, at
|
|
5
|
+
// tier 4 66%, at tier 5 56%. Output is bought by turns, and every other system here targets
|
|
6
|
+
// input. At tier 4 `specpi-jev` runs cheaper per turn than `specpi-default` -- $0.000538 against
|
|
7
|
+
// $0.000613 -- and costs more per attempt anyway, because it takes 29.4 turns against 24.8.
|
|
8
|
+
//
|
|
9
|
+
// The evidence that turns are wasted rather than merely numerous is in the same runs.
|
|
10
|
+
// `specpi-default` invoked `webqa` 199 times across 30 turns on `t4-browser-triage` and still
|
|
11
|
+
// failed. Pi stock abandoned after two tries and burned the full 600-second timeout on two tier-4
|
|
12
|
+
// tasks, scoring zero both times. A turn costs 4-7 seconds and $0.0005-$0.0011; a Jev call costs
|
|
13
|
+
// about 300ms and $0.0000105. Preventing one timeout pays for roughly two thousand calls.
|
|
14
|
+
//
|
|
15
|
+
// LOCAL STATE FIRST, which is the standing rule and the reason this is affordable. Local state
|
|
16
|
+
// cannot answer "is this session stuck" -- that needs a judgement about whether the work is going
|
|
17
|
+
// anywhere -- but it answers "is that question worth asking", and it answers it cheaply. Nothing is
|
|
18
|
+
// sent unless a repeated tool signature, a run of errors or a stretch of turns without a file
|
|
19
|
+
// change has already made the session look suspicious.
|
|
20
|
+
//
|
|
21
|
+
// It is an append, so it passes the standing cache rule: the line goes on the end of the transcript
|
|
22
|
+
// at a turn boundary and invalidates no prefix. It is written once and never retracted, because
|
|
23
|
+
// retracting it would mean rewriting the thing it was appended to.
|
|
24
|
+
//
|
|
25
|
+
// The plan named `deliverAs: "nextTurn"` for that append. Pi documents that mode as "queued for
|
|
26
|
+
// next user prompt, does not interrupt or trigger anything", and an unattended session has exactly
|
|
27
|
+
// one user prompt -- so a nextTurn message would never arrive, in precisely the case the argument
|
|
28
|
+
// for this system rests on. The caller uses "steer" instead: delivered after the current tool calls
|
|
29
|
+
// finish and before the next model request, which is the same append at the same boundary and is
|
|
30
|
+
// actually read. `triggerTurn` stays off, so this can never add a turn of its own.
|
|
31
|
+
|
|
32
|
+
import { choice, noul } from "../client.mjs";
|
|
33
|
+
import { choiceValue, nounTrue } from "../gate.mjs";
|
|
34
|
+
import { compact } from "../sanitize.mjs";
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* The same taxonomy `scripts/jev-triage.mjs` uses offline. It lives here, in the shipped extension,
|
|
38
|
+
* and the script imports it: the corpus that calibrates this system and the enum this system asks
|
|
39
|
+
* against have to be the same object, or the published failure-mode distribution is measuring
|
|
40
|
+
* something the session never asks.
|
|
41
|
+
*/
|
|
42
|
+
export const FAILURE_MODES = Object.freeze({
|
|
43
|
+
"gave-up-on-fault": "Met an injected command failure and stopped instead of retrying",
|
|
44
|
+
"turn-cap": "Ran out of turns or requests while still working",
|
|
45
|
+
timeout: "Exceeded the wall-clock limit",
|
|
46
|
+
"wrong-approach": "Worked steadily but solved the wrong problem",
|
|
47
|
+
"misread-requirement": "Produced output that misses a stated requirement",
|
|
48
|
+
"scope-violation": "Changed files it was told to leave alone",
|
|
49
|
+
"tool-error-loop": "Repeated the same failing tool call without progress",
|
|
50
|
+
"harness-error": "The harness itself crashed or could not start",
|
|
51
|
+
unknown: "Not determinable from what was recorded",
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
/** Modes a session can still act on. A timeout or a crashed harness is not advice, it is an epitaph. */
|
|
55
|
+
export const ACTIONABLE_MODES = Object.freeze(
|
|
56
|
+
new Set(["wrong-approach", "misread-requirement", "tool-error-loop", "gave-up-on-fault", "scope-violation"]),
|
|
57
|
+
);
|
|
58
|
+
|
|
59
|
+
/** Tools whose success means the worktree changed. Used only to decide whether anything happened. */
|
|
60
|
+
export const MUTATING_TOOLS = Object.freeze(
|
|
61
|
+
new Set(["write", "edit", "multi_edit", "apply_patch", "create_file", "str_replace"]),
|
|
62
|
+
);
|
|
63
|
+
|
|
64
|
+
/** Turns without a single successful mutation before that counts as a reason to look. */
|
|
65
|
+
export const STALE_TURNS = 4;
|
|
66
|
+
const REPEAT_SIGNATURES = 2;
|
|
67
|
+
const CONSECUTIVE_ERRORS = 3;
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Turns to stay quiet after asking. Conditions persist for many turns at a time, so without this
|
|
71
|
+
* the same unchanged situation is re-asked every turn until the budget runs out.
|
|
72
|
+
*/
|
|
73
|
+
export const ASK_COOLDOWN_TURNS = 4;
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* A tool call reduced to something comparable. The arguments matter -- reading two different files
|
|
77
|
+
* twice is work, reading one file twice is a loop -- but their contents do not, so this keeps a
|
|
78
|
+
* short normalized form rather than the payload.
|
|
79
|
+
*/
|
|
80
|
+
export function signature(toolName, input) {
|
|
81
|
+
return compact(`${toolName}:${JSON.stringify(input ?? {})}`, 120);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Whether local state alone already justifies the question. Returns the reasons rather than a bare
|
|
86
|
+
* boolean, because they go into the state as the grounds for asking and a caller logging "asked"
|
|
87
|
+
* without "why" cannot audit the budget afterwards.
|
|
88
|
+
*
|
|
89
|
+
* One reason is not enough, and that is a correction rather than a precaution. The first version
|
|
90
|
+
* asked whenever any single signal fired, and a live run of `t3-cascade-ledger` spent all twelve
|
|
91
|
+
* calls of its budget on a session that finished with a score of 0.978. A 120-step repair chain
|
|
92
|
+
* re-runs the same verification command constantly, so "a tool signature repeated" is its normal
|
|
93
|
+
* condition, not a symptom. The same is true of "no file written for four turns" during a long
|
|
94
|
+
* read: a research task looks identical to a stuck one from that signal alone.
|
|
95
|
+
*
|
|
96
|
+
* So a repeated call or a quiet stretch must coincide with something else before it is worth
|
|
97
|
+
* asking about. A run of consecutive errors stands alone, because nothing healthy produces three
|
|
98
|
+
* failures in a row.
|
|
99
|
+
*/
|
|
100
|
+
export function suspicious(history) {
|
|
101
|
+
const reasons = [];
|
|
102
|
+
const signatures = history?.signatures ?? [];
|
|
103
|
+
const counts = new Map();
|
|
104
|
+
for (const item of signatures) {
|
|
105
|
+
counts.set(item, (counts.get(item) ?? 0) + 1);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
const repeated = [...counts.entries()].filter(([, count]) => count >= REPEAT_SIGNATURES);
|
|
109
|
+
if (repeated.length > 0) {
|
|
110
|
+
reasons.push("repeated-tool-call");
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const errors = (history?.consecutiveErrors ?? 0) >= CONSECUTIVE_ERRORS;
|
|
114
|
+
if (errors) {
|
|
115
|
+
reasons.push("consecutive-errors");
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
if ((history?.turnsSinceChange ?? 0) >= STALE_TURNS) {
|
|
119
|
+
reasons.push("no-file-change");
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// Conditions persist across turns, so a verdict of "not stuck" is still the right answer three
|
|
123
|
+
// turns later and asking again buys nothing. The cooldown is what stops one situation being
|
|
124
|
+
// charged for repeatedly.
|
|
125
|
+
const cooling = Number.isFinite(history?.askedAtTurn)
|
|
126
|
+
? (history.turn ?? 0) - history.askedAtTurn < ASK_COOLDOWN_TURNS
|
|
127
|
+
: false;
|
|
128
|
+
|
|
129
|
+
return {
|
|
130
|
+
ask: !cooling && (errors || reasons.length >= 2),
|
|
131
|
+
reasons,
|
|
132
|
+
repeatedSignatures: repeated.length,
|
|
133
|
+
cooling,
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** Shape and counts. No tool output, no file contents, no arguments beyond a normalized signature. */
|
|
138
|
+
export function buildInput({ history, objective, reasons }) {
|
|
139
|
+
return {
|
|
140
|
+
objective: compact(objective ?? "", 180),
|
|
141
|
+
turn: history?.turn ?? 0,
|
|
142
|
+
reasons,
|
|
143
|
+
turnsSinceFileChange: history?.turnsSinceChange ?? 0,
|
|
144
|
+
consecutiveErrors: history?.consecutiveErrors ?? 0,
|
|
145
|
+
distinctTools: [...new Set((history?.tools ?? []).slice(-12))].slice(0, 12),
|
|
146
|
+
repeatedCalls: suspicious(history).repeatedSignatures,
|
|
147
|
+
recentErrors: (history?.errors ?? []).slice(-4).map((item) => compact(item, 80)),
|
|
148
|
+
filesChanged: history?.filesChanged ?? 0,
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
export function questions() {
|
|
153
|
+
return {
|
|
154
|
+
// Deliberately two questions rather than one. "Is it stuck" is the decision; "which mode"
|
|
155
|
+
// is what makes a fixed remedy line possible without any model-written prose.
|
|
156
|
+
is_stuck: noul("This session has stopped making progress and will not finish without changing approach"),
|
|
157
|
+
failure_mode: choice("If this session is going to fail, why?", FAILURE_MODES),
|
|
158
|
+
needs_human: noul("A person would have to answer something before this session could continue"),
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** One fixed line per mode. Code-written, so nothing the model produced reaches the transcript. */
|
|
163
|
+
const REMEDIES = Object.freeze({
|
|
164
|
+
"wrong-approach":
|
|
165
|
+
"Progress check: the last few turns have not moved the task forward. Re-read the objective and state, in one line, what the current approach is meant to achieve before continuing.",
|
|
166
|
+
"misread-requirement":
|
|
167
|
+
"Progress check: the work so far may not match what was asked. Re-read the requirements and list which ones are satisfied and which are not, before making further changes.",
|
|
168
|
+
"tool-error-loop":
|
|
169
|
+
"Progress check: the same tool call has failed repeatedly. Stop retrying it and either fix the cause or use a different approach.",
|
|
170
|
+
"gave-up-on-fault":
|
|
171
|
+
"Progress check: a command failed and was not retried. Transient failures are expected here; retry it before concluding the task cannot be done.",
|
|
172
|
+
"scope-violation":
|
|
173
|
+
"Progress check: files outside the permitted scope may have been changed. Check what has been modified against what the task allows.",
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* The nudge, or nothing. Requires a confident stuck verdict AND a gated mode with a remedy: a
|
|
178
|
+
* confident "stuck" with no idea why would produce a message that says only that something is
|
|
179
|
+
* wrong, which is the kind of unfalsifiable hint the standing rule against risk hints rejects.
|
|
180
|
+
*
|
|
181
|
+
* `needsHuman` decides who the nudge is for. If the session is blocked on something only a person
|
|
182
|
+
* can answer -- a missing credential, an ambiguous requirement -- then telling the model to try
|
|
183
|
+
* harder is the wrong recipient and costs a turn to say nothing. The caller reads it to suppress
|
|
184
|
+
* the message path while still surfacing the notification.
|
|
185
|
+
*/
|
|
186
|
+
export function decide(answers) {
|
|
187
|
+
const stuck = nounTrue(answers?.is_stuck, "progress");
|
|
188
|
+
const mode = choiceValue(answers?.failure_mode, "progress");
|
|
189
|
+
const needsHuman = nounTrue(answers?.needs_human, "progress");
|
|
190
|
+
if (!stuck || !mode || !ACTIONABLE_MODES.has(mode) || !REMEDIES[mode]) {
|
|
191
|
+
return { nudge: undefined, stuck, mode, needsHuman };
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
return { nudge: REMEDIES[mode], stuck, mode, needsHuman };
|
|
195
|
+
}
|