sumo-agents 0.7.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +88 -0
- package/README.md +43 -20
- package/package.json +2 -1
- package/prompts/scribe.md +1 -1
- package/src/apply.mjs +16 -12
- package/src/auth.mjs +2 -2
- package/src/card.mjs +3 -2
- package/src/catalog.mjs +100 -13
- package/src/chat.mjs +71 -16
- package/src/check.mjs +17 -15
- package/src/cli.mjs +21 -11
- package/src/cost.mjs +2 -4
- package/src/cursor.mjs +405 -0
- package/src/db.mjs +24 -13
- package/src/dream.mjs +2 -2
- package/src/jobs.mjs +32 -14
- package/src/ledger.mjs +4 -3
- package/src/loop.mjs +124 -73
- package/src/mcp.mjs +7 -9
- package/src/memeval.mjs +2 -3
- package/src/memory.mjs +5 -4
- package/src/model.mjs +5 -0
- package/src/page.mjs +1 -0
- package/src/prime.mjs +11 -7
- package/src/rebuild.mjs +8 -3
- package/src/redact.mjs +13 -1
- package/src/route.mjs +263 -56
- package/src/sandbox.mjs +86 -20
- package/src/scan.mjs +4 -2
- package/src/scorecard.mjs +419 -0
- package/src/scribe.mjs +82 -38
- package/src/sessions.mjs +9 -7
- package/src/settings.mjs +114 -37
- package/src/setup.mjs +33 -14
- package/src/spool.mjs +68 -43
- package/src/tools.mjs +170 -77
- package/src/ui.mjs +14 -5
- package/src/verify.mjs +68 -12
- package/src/worktree.mjs +23 -16
package/src/route.mjs
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { CURSOR_MODELS, JOB_MODELS, modelId, usableJobModels, usableModels } from './catalog.mjs';
|
|
2
2
|
import { getMeta } from './db.mjs';
|
|
3
|
+
import { routeScore, savedScorecard } from './scorecard.mjs';
|
|
3
4
|
import { CONFIG_DEFAULTS } from './setup.mjs';
|
|
4
|
-
import { callLocalModel } from './model.mjs';
|
|
5
|
+
import { callLocalModel, costOf } from './model.mjs';
|
|
5
6
|
import { UsageError } from './memory.mjs';
|
|
6
7
|
import { elapsed } from './text.mjs';
|
|
7
8
|
|
|
@@ -12,31 +13,43 @@ export const RETRY_IMPORTANT_THRESHOLD = 3;
|
|
|
12
13
|
const KINDS = ['mechanical', 'routine', 'hard', 'novel'];
|
|
13
14
|
const SURFACES = ['one spot', 'a few files', 'many interacting parts'];
|
|
14
15
|
|
|
16
|
+
/**
|
|
17
|
+
* Each router prompt: who is being routed, the kinds to sort the work into, and the rule from kind and surface to a route.
|
|
18
|
+
* With a scorecard saved (src/scorecard.mjs) the same prompt is asked for the sorting alone, and code picks the route
|
|
19
|
+
* (`pickRoute`); with none, the router still chooses by the rule, as it always has. The rule stays in the prompt either way:
|
|
20
|
+
* measured on the router probe (Nov 18 2026), a prompt without it made the local model call a one-function change
|
|
21
|
+
* "many interacting parts" and risky, and the same prompt with it sorted the change as routine, in one spot.
|
|
22
|
+
*/
|
|
23
|
+
const JOB_KINDS =
|
|
24
|
+
'kind: mechanical (typos, renames, comments, constants, tests for a pure function), routine (an ordinary feature or fix with a clear spec), ' +
|
|
25
|
+
'hard (design, a bug whose cause is unknown, or reviewing risky code), novel (new algorithms, subtle concurrency, deep security review). ' +
|
|
26
|
+
'surface: how much code the job touches. risky: true when it touches security, credentials, money, migrations or concurrency.';
|
|
15
27
|
const ROUTER_SYSTEM =
|
|
16
|
-
'You assign a coding job to a model and an effort level. Answer with JSON only. First classify the job, then choose.
|
|
17
|
-
|
|
18
|
-
'
|
|
19
|
-
'
|
|
20
|
-
'
|
|
21
|
-
'files, high for many interacting parts. risky raises the effort one step (low, medium, high, xhigh, max) and makes a routine job hard. ' +
|
|
22
|
-
'History lists only the routes that failed or drew Important review findings on this project: for work like that, go above them.';
|
|
28
|
+
'You assign a coding job to a model and an effort level. Answer with JSON only. First classify the job, then choose. ' +
|
|
29
|
+
JOB_KINDS +
|
|
30
|
+
' Model follows kind: haiku for mechanical, sonnet for routine, opus for hard, fable for novel. Effort follows surface: low for one spot, ' +
|
|
31
|
+
'medium for a few files, high for many interacting parts. risky raises the effort one step (low, medium, high, xhigh, max) and makes a ' +
|
|
32
|
+
'routine job hard. History lists only the routes that failed or drew Important review findings on this project: for work like that, go above them.';
|
|
23
33
|
|
|
24
34
|
/**
|
|
25
35
|
* The same grammar read for a scout, which reads and reports and changes nothing: the kinds above are told in terms of
|
|
26
36
|
* changing code, and read that way an audit is "novel" and a summary of an unfamiliar project is too — the dearest model
|
|
27
37
|
* for work that is reading. Here they are told in terms of what a reading job asks for.
|
|
28
38
|
*/
|
|
39
|
+
const SCOUT_KINDS =
|
|
40
|
+
'kind: mechanical (one lookup: where one thing is defined, what one setting is), routine (finding, tracing, listing, describing or ' +
|
|
41
|
+
'summarizing code: what a project does, how a feature works, which endpoints exist, which dependencies are behind — however large or ' +
|
|
42
|
+
'unfamiliar the project), hard (it must judge: an audit of security or correctness, a review, a hunt for the cause of a bug), novel ' +
|
|
43
|
+
'(almost never: only reasoning nobody has written down, such as proving a concurrent algorithm correct). surface: how much code must be ' +
|
|
44
|
+
'read. risky: true when what is judged is security, credentials, money, migrations or concurrency.';
|
|
45
|
+
const SCOUT = 'The job is done by a scout: it reads code and reports, and changes nothing.';
|
|
29
46
|
const SCOUT_ROUTER_SYSTEM =
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
'
|
|
33
|
-
'
|
|
34
|
-
'
|
|
35
|
-
'
|
|
36
|
-
'migrations or concurrency. Model follows kind: haiku for mechanical, sonnet for routine, opus for hard, ' +
|
|
37
|
-
'fable for novel. Effort follows surface: low for one spot, medium for a few files or a whole project read for a summary, high when ' +
|
|
38
|
-
'many interacting parts must be judged together. risky raises the effort one step (low, medium, high, xhigh, max). ' +
|
|
39
|
-
'History lists only the routes that failed or drew Important review findings on this project: for work like that, go above them.';
|
|
47
|
+
`You assign a reading job to a model and an effort level. ${SCOUT} Answer with JSON only. First classify the job, then choose. ` +
|
|
48
|
+
SCOUT_KINDS +
|
|
49
|
+
' Model follows kind: haiku for mechanical, sonnet for routine, opus for hard, fable for novel. Effort follows surface: low for one spot, ' +
|
|
50
|
+
'medium for a few files or a whole project read for a summary, high when many interacting parts must be judged together. risky raises the ' +
|
|
51
|
+
'effort one step (low, medium, high, xhigh, max). History lists only the routes that failed or drew Important review findings on this ' +
|
|
52
|
+
'project: for work like that, go above them.';
|
|
40
53
|
|
|
41
54
|
/**
|
|
42
55
|
* The answer shape: the models that are on and the efforts every one of them takes. The classification comes
|
|
@@ -58,6 +71,172 @@ const answerShape = (models, efforts) => ({
|
|
|
58
71
|
});
|
|
59
72
|
/** The grammar over the models that are on. */
|
|
60
73
|
const routerSchema = (usable) => answerShape(usable, EFFORTS);
|
|
74
|
+
/** The grammar when the router only sorts: the classification and its reason, no route. */
|
|
75
|
+
const CLASSIFY_SCHEMA = {
|
|
76
|
+
type: 'object',
|
|
77
|
+
properties: {
|
|
78
|
+
kind: { type: 'string', enum: KINDS },
|
|
79
|
+
surface: { type: 'string', enum: SURFACES },
|
|
80
|
+
risky: { type: 'boolean' },
|
|
81
|
+
reason: { type: 'string', maxLength: 200 },
|
|
82
|
+
},
|
|
83
|
+
required: ['kind', 'surface', 'risky', 'reason'],
|
|
84
|
+
additionalProperties: false,
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* What each kind asks of a route's score on the scorecard (0–1, a place among Sumo's models): starting values. The effort
|
|
89
|
+
* still follows the surface, as the rule had it, and a larger or risky ask needs at least what the smaller one got (`pickRoute`).
|
|
90
|
+
* On the scorecard of Nov 18 2026 a job gets haiku for mechanical work and sonnet for routine, as the rule had it; hard work,
|
|
91
|
+
* risky work and novel work get opus, which scores above the catalog's Fable 5.1 (for hard work in one spot sonnet at high
|
|
92
|
+
* effort clears the bar too, and costs more than opus at low). The chat, open to xhigh, gets sonnet at xhigh for hard work past
|
|
93
|
+
* one spot: it scores as much as opus at low, for less than opus at medium.
|
|
94
|
+
*/
|
|
95
|
+
const BARS = { mechanical: 0, routine: 0.3, hard: 0.55, novel: 0.75 };
|
|
96
|
+
const SURFACE_EFFORT = { 'one spot': 'low', 'a few files': 'medium', 'many interacting parts': 'high' };
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* A route as the run it is: a model that takes no effort (Composer) runs the same at every one, and one on Cursor that takes
|
|
100
|
+
* an effort is sent its xhigh for max (`cursorModel`, src/cursor.mjs). What the ledger knows of a run, and what this project's
|
|
101
|
+
* record holds against it, is known of each route that is that run.
|
|
102
|
+
*/
|
|
103
|
+
function runOf(model, effort) {
|
|
104
|
+
const cursor = CURSOR_MODELS[model];
|
|
105
|
+
if (!cursor) return `${model}/${effort}`;
|
|
106
|
+
return cursor.effort ? `${model}/${effort === 'max' ? 'xhigh' : effort}` : `${model}/*`;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* The cheapest of `routes` by one measure: their prices, when every one of them has one — never some priced and some not —
|
|
111
|
+
* and on equal prices, or with none, the lower rung: the model lower on the job ladder, then the lower effort.
|
|
112
|
+
*/
|
|
113
|
+
function cheapest(routes) {
|
|
114
|
+
const priced = routes.every((r) => r.price);
|
|
115
|
+
return [...routes].sort((a, b) => (priced && a.price.cost !== b.price.cost ? a.price.cost - b.price.cost : 0) || compareRoutes(a, b))[0];
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* The route code picks from the router's sorting and the saved scorecard: the cheapest route — priced from the `ledger` where
|
|
120
|
+
* it can (`priceRoutes`), else by the benchmarks — whose score clears the bar for the kind, at the effort the surface asks for
|
|
121
|
+
* or above, one step more when risky, and no higher than `highest`. risky also makes routine work hard where `hardens`. The
|
|
122
|
+
* floors are taken in turn from the lowest, each needing what the pick one floor down scored, so a larger or risky ask never
|
|
123
|
+
* gets a lower-scoring route than a smaller one of the same kind. A retry passes `above`, the score of the attempt retried:
|
|
124
|
+
* the route must score more, and says whether one did (`climbed`). A route in `avoid` (this project's record is against it)
|
|
125
|
+
* is passed over while another is scored. When none clears the bar, the best-scored route. Null when no route of the
|
|
126
|
+
* `usable` models is on the scorecard. Pure.
|
|
127
|
+
*/
|
|
128
|
+
export function pickRoute({ card, usable, kind, surface, risky = false, hardens = true, highest = null, avoid = [], ledger = null, above = null }) {
|
|
129
|
+
const asked = risky && hardens && kind === 'routine' ? 'hard' : kind;
|
|
130
|
+
const top = highest ? EFFORTS.indexOf(highest) : EFFORTS.length - 1;
|
|
131
|
+
const floor = Math.min(EFFORTS.indexOf(SURFACE_EFFORT[surface] ?? 'low') + (risky ? 1 : 0), top);
|
|
132
|
+
// One route per run (`runOf`): Composer at the floor's effort, and no max on Cursor — either would be the same job said again.
|
|
133
|
+
const efforts = (model, at) => {
|
|
134
|
+
const cursor = CURSOR_MODELS[model];
|
|
135
|
+
if (cursor && !cursor.effort) return [EFFORTS[at]];
|
|
136
|
+
return EFFORTS.slice(at, top + 1).filter((effort) => !cursor || effort !== 'max');
|
|
137
|
+
};
|
|
138
|
+
const avoided = new Set(avoid.map((route) => runOf(...route.split('/'))));
|
|
139
|
+
const poolAt = (at) => {
|
|
140
|
+
const scored = priceRoutes(usable.flatMap((model) => efforts(model, at).map((effort) => ({ model, effort, on: routeScore(card, model, effort, EFFORTS) }))).filter((r) => r.on), ledger, card);
|
|
141
|
+
const open = scored.filter((r) => !avoided.has(runOf(r.model, r.effort)));
|
|
142
|
+
return open.length > 0 ? open : scored;
|
|
143
|
+
};
|
|
144
|
+
let bar = BARS[asked];
|
|
145
|
+
let raisedBy = null;
|
|
146
|
+
let pick = null;
|
|
147
|
+
let clear = [];
|
|
148
|
+
for (let at = 0; at <= floor; at += 1) {
|
|
149
|
+
const pool = poolAt(at);
|
|
150
|
+
if (pool.length === 0) return null;
|
|
151
|
+
if (pick && pick.on.score > bar) [bar, raisedBy] = [pick.on.score, pick];
|
|
152
|
+
clear = pool.filter((r) => r.on.score >= bar && (above === null || r.on.score > above));
|
|
153
|
+
pick = clear.length > 0 ? cheapest(clear) : [...pool].sort((a, b) => b.on.score - a.on.score || compareRoutes(a, b))[0];
|
|
154
|
+
}
|
|
155
|
+
const two = (n) => Number(n.toFixed(2));
|
|
156
|
+
const needs = `${asked}${risky ? ', risky' : ''}, ${surface} needs ${two(bar)}${raisedBy ? `, what ${raisedBy.model}/${raisedBy.effort} scores a step smaller` : ''}${above === null ? '' : ` and more than ${two(above)}, what the attempt retried scores`}`;
|
|
157
|
+
const price = clear.length > 0 && clear.every((r) => r.price) ? `$${pick.price.cost.toFixed(2)} ${pick.price.per}, the cheapest` : 'an unknown cost, the lowest';
|
|
158
|
+
const why = clear.length > 0
|
|
159
|
+
? `${needs}; ${pick.model}/${pick.effort} scores ${pick.on.score.toFixed(2)} at ${price} of ${clear.length} that do`
|
|
160
|
+
: `${needs}; none does, and ${pick.model}/${pick.effort} scores best at ${pick.on.score.toFixed(2)}`;
|
|
161
|
+
return { model: pick.model, effort: pick.effort, why, ...(above === null ? {} : { climbed: clear.length > 0 }) };
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* What Sumo's own jobs of one `agent` have cost, from the ledger, for pricing routes: each run's finished jobs (`runOf`) — how many,
|
|
166
|
+
* and their average — and the typical finished job's tokens, read from the cache and not. Only a job that did billed work
|
|
167
|
+
* counts: one that failed at its first request cost nothing and says nothing of its route. A scout's reading prices a scout's
|
|
168
|
+
* route, never a worker's. Neither API gives a cost back here (Cursor's needs a feature this account lacks), and the ledger
|
|
169
|
+
* prices every request as it comes: it is the one measure of what Sumo's own work costs. Null while it holds no such job.
|
|
170
|
+
*/
|
|
171
|
+
export function ledgerCost(db, agent) {
|
|
172
|
+
const jobs = db
|
|
173
|
+
.prepare(
|
|
174
|
+
`SELECT j.model AS model, j.effort AS effort, SUM(r.cost_usd) AS cost, SUM(r.input_tokens) AS input, SUM(COALESCE(r.cache_read_tokens, 0)) AS cached,
|
|
175
|
+
SUM(COALESCE(r.cache_creation_tokens, 0)) AS written, SUM(r.output_tokens) AS output
|
|
176
|
+
FROM jobs j JOIN model_runs r ON r.job_id = j.id
|
|
177
|
+
WHERE j.model IS NOT NULL AND j.status IN ('done', 'failed') AND j.agent = ? GROUP BY j.id HAVING SUM(r.cost_usd) > 0`,
|
|
178
|
+
)
|
|
179
|
+
.all(agent);
|
|
180
|
+
if (jobs.length === 0) return null;
|
|
181
|
+
const mean = (k) => jobs.reduce((sum, j) => sum + (j[k] ?? 0), 0) / jobs.length;
|
|
182
|
+
const own = new Map();
|
|
183
|
+
for (const j of jobs) {
|
|
184
|
+
const run = runOf(j.model, j.effort);
|
|
185
|
+
const seen = own.get(run) ?? { sum: 0, n: 0 };
|
|
186
|
+
own.set(run, { sum: seen.sum + j.cost, n: seen.n + 1 });
|
|
187
|
+
}
|
|
188
|
+
return {
|
|
189
|
+
own: new Map([...own].map(([route, { sum, n }]) => [route, { cost: sum / n, jobs: n }])),
|
|
190
|
+
typical: { input: mean('input'), cached: mean('cached'), written: mean('written'), output: mean('output') },
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Prices each route for picking, in one unit. With a `ledger` (`ledgerCost`), what a job costs: a route's own average over its
|
|
196
|
+
* jobs; else the benchmarks' cost of one of their tasks scaled to the size of Sumo's jobs — by how the routes the ledger has
|
|
197
|
+
* priced compare with what the benchmarks say of them — so a dearer effort or model stays dearer; else the typical job's
|
|
198
|
+
* tokens at the model's prices. Without one, what a benchmark task costs, where a source says. A price that is not above
|
|
199
|
+
* zero is none. `routes` are `{ model, effort, on }`; the scale is taken over every route the ledger has priced and the
|
|
200
|
+
* `card` prices too, whether or not this job may use it.
|
|
201
|
+
*/
|
|
202
|
+
function priceRoutes(routes, ledger, card) {
|
|
203
|
+
const priced = (cost, per) => (cost > 0 ? { cost, per } : null);
|
|
204
|
+
if (!ledger) return routes.map((r) => ({ ...r, price: priced(r.on.cost, 'a task on the benchmarks') }));
|
|
205
|
+
const ratios = [...ledger.own]
|
|
206
|
+
.map(([route, mine]) => [mine, routeScore(card, ...route.split('/'), EFFORTS)?.cost])
|
|
207
|
+
.filter(([, cost]) => cost > 0)
|
|
208
|
+
.map(([mine, cost]) => mine.cost / cost)
|
|
209
|
+
.filter((ratio) => ratio > 0)
|
|
210
|
+
.sort((a, b) => a - b);
|
|
211
|
+
const scale = ratios.length > 0 ? ratios[Math.floor(ratios.length / 2)] : null;
|
|
212
|
+
const typical = (model) => {
|
|
213
|
+
const t = ledger.typical;
|
|
214
|
+
return costOf(modelId(model), { input_tokens: Math.max(0, t.input - t.cached - t.written), cache_read_input_tokens: t.cached, cache_creation_input_tokens: t.written, output_tokens: t.output });
|
|
215
|
+
};
|
|
216
|
+
return routes.map((r) => {
|
|
217
|
+
const mine = ledger.own.get(runOf(r.model, r.effort));
|
|
218
|
+
if (mine) return { ...r, price: priced(mine.cost, `a job, by its ${mine.jobs} in the ledger`) };
|
|
219
|
+
if (r.on.cost > 0 && scale !== null) return { ...r, price: priced(r.on.cost * scale, `a job, from the benchmarks scaled to the ledger's (×${scale.toPrecision(2)})`) };
|
|
220
|
+
return { ...r, price: priced(typical(r.model), "a job, the ledger's typical one at its prices") };
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* The saved scorecard, when it lists any of `usable` and scores no other version of one of them: what decides whether the router
|
|
226
|
+
* sorts or chooses. One fetched before the catalog moved a model on is set aside until Discover fetches it again — read, it would
|
|
227
|
+
* leave that model out of routing for want of a score.
|
|
228
|
+
*/
|
|
229
|
+
function scorecardFor(db, usable) {
|
|
230
|
+
const card = savedScorecard(db);
|
|
231
|
+
const stale = Object.values(card?.sources ?? {}).some((s) => s.entries.some((e) => usable.includes(e.model) && e.id !== modelId(e.model)));
|
|
232
|
+
return card && !stale && usable.some((m) => EFFORTS.some((e) => routeScore(card, m, e, EFFORTS))) ? card : null;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** The routes this project's record is against: half or more of their jobs failed, or their reviews averaged enough Important findings. */
|
|
236
|
+
const troubled = (db, projectSlug) =>
|
|
237
|
+
routeStats(db, { project: projectSlug })
|
|
238
|
+
.filter((r) => (r.failed > 0 && r.failed >= r.done) || (r.avgImportant ?? 0) >= RETRY_IMPORTANT_THRESHOLD)
|
|
239
|
+
.map((r) => `${r.model}/${r.effort}`);
|
|
61
240
|
|
|
62
241
|
const TASK_CUT = 3500;
|
|
63
242
|
|
|
@@ -140,15 +319,15 @@ function retryLine(db, retryOf) {
|
|
|
140
319
|
return `retry of j${retryOf}: it ran on ${prev.model ?? 'no recorded route'}${prev.effort ? `/${prev.effort}` : ''} and ${outcome}`;
|
|
141
320
|
}
|
|
142
321
|
|
|
143
|
-
|
|
322
|
+
/** What the router reads of a job. Only sorting it (`classify`), it is not shown the models or their history: code weighs those. */
|
|
323
|
+
function routerPrompt(db, { agent, project, task, title, retryOf }, { classify = false } = {}) {
|
|
144
324
|
const stack = stackOf(db, project.slug);
|
|
145
325
|
return [
|
|
146
326
|
`role: ${agent}`,
|
|
147
327
|
`project: ${project.slug}${stack ? ` (${stack})` : ''}`,
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
historyText(db, project.slug),
|
|
328
|
+
...(classify
|
|
329
|
+
? []
|
|
330
|
+
: [`models: ${jobModels(db, { claude: true }).join(', ')}`, ...(retryOf === undefined ? [] : [retryLine(db, retryOf)].filter(Boolean)), 'history:', historyText(db, project.slug)]),
|
|
152
331
|
`title: ${title ?? ''}`,
|
|
153
332
|
'',
|
|
154
333
|
String(task ?? '').slice(0, TASK_CUT),
|
|
@@ -159,52 +338,64 @@ function routerPrompt(db, { agent, project, task, title, retryOf }) {
|
|
|
159
338
|
* The same grammar read for a chat turn: the message is the whole job, there is no brief to size it by
|
|
160
339
|
* and no history of turns, so the kinds are told in terms of what a message asks for.
|
|
161
340
|
*/
|
|
341
|
+
const CHAT_KINDS =
|
|
342
|
+
'kind: mechanical (a one-line question, a lookup, a typo, a rename, running a command and reporting), routine (an ordinary feature or fix ' +
|
|
343
|
+
'with a clear ask, a summary, an explanation), hard (design, a bug whose cause is unknown, a review of risky code, planning across a ' +
|
|
344
|
+
'codebase), novel (new algorithms, subtle concurrency, deep security review). surface: how much code the turn touches. risky: true when ' +
|
|
345
|
+
'it touches security, credentials, money, migrations or concurrency.';
|
|
346
|
+
const CHAT = 'The chat is a coding assistant with a shell and an editor; the turn is what the user just typed.';
|
|
162
347
|
const CHAT_ROUTER_SYSTEM =
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
'
|
|
166
|
-
'a
|
|
167
|
-
'
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
'xhigh, max) and makes a routine turn hard.';
|
|
171
|
-
|
|
172
|
-
function chatRouterPrompt(db, { project, text }) {
|
|
348
|
+
`You assign one chat turn to a model and an effort level. ${CHAT} Answer with JSON only. First classify the turn, then choose. ` +
|
|
349
|
+
CHAT_KINDS +
|
|
350
|
+
' Model follows kind: haiku for mechanical, sonnet for routine, opus for hard, fable for novel. Effort follows surface: low for one spot, ' +
|
|
351
|
+
'medium for a few files, high for many interacting parts. risky raises the effort one step (low, medium, high, xhigh, max) and makes a ' +
|
|
352
|
+
'routine turn hard.';
|
|
353
|
+
|
|
354
|
+
function chatRouterPrompt(db, { project, text }, { classify = false } = {}) {
|
|
173
355
|
const stack = project ? stackOf(db, project.slug) : null;
|
|
174
|
-
|
|
356
|
+
const models = classify ? [] : [`models: ${usableModels(db).join(', ')}`];
|
|
357
|
+
return [`project: ${project ? `${project.slug}${stack ? ` (${stack})` : ''}` : 'none'}`, ...models, 'turn:', String(text ?? '').slice(0, TASK_CUT)].join('\n');
|
|
175
358
|
}
|
|
176
359
|
|
|
177
360
|
/** The smallest model a job may run on (`jobs.floor`); haiku, the cheapest, holds nothing back. */
|
|
178
361
|
const jobFloor = (db) => getMeta(db, 'config.jobs.floor') ?? CONFIG_DEFAULTS['jobs.floor'];
|
|
179
362
|
|
|
180
|
-
/**
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
363
|
+
/**
|
|
364
|
+
* The models a job may be routed to: the ones that are on, from the floor up — or all of them, when none at or above it is on.
|
|
365
|
+
* Cursor's are among them where the scorecard weighs them; the rule names Claude's alone, so where the router chooses by it
|
|
366
|
+
* (`claude`) only Claude's are.
|
|
367
|
+
*/
|
|
368
|
+
export function jobModels(db, { claude = false } = {}) {
|
|
369
|
+
const usable = claude ? usableModels(db) : usableJobModels(db);
|
|
370
|
+
const above = usable.filter((m) => JOB_MODELS.indexOf(m) >= JOB_MODELS.indexOf(jobFloor(db)));
|
|
184
371
|
return above.length > 0 ? above : usable;
|
|
185
372
|
}
|
|
186
373
|
|
|
187
|
-
const askRouter = (db, { agent, project, task, title, retryOf }
|
|
374
|
+
const askRouter = (db, { agent, project, task, title, retryOf }, { classify = false } = {}) =>
|
|
375
|
+
askFor(db, agent === 'scout' ? SCOUT_ROUTER_SYSTEM : ROUTER_SYSTEM, routerPrompt(db, { agent, project, task, title, retryOf }, { classify }), jobModels(db, { claude: !classify }), { classify });
|
|
188
376
|
|
|
189
|
-
async function askFor(db, system, prompt, models = usableModels(db)) {
|
|
377
|
+
async function askFor(db, system, prompt, models = usableModels(db), { classify = false } = {}) {
|
|
190
378
|
const usable = models;
|
|
191
379
|
if (usable.length === 0) throw new UsageError('no model is on — turn one on under API models, on the settings page (sumo config)');
|
|
192
|
-
const result = await callLocalModel(db, { system, prompt, schema: routerSchema(usable) });
|
|
380
|
+
const result = await callLocalModel(db, { system, prompt, schema: classify ? CLASSIFY_SCHEMA : routerSchema(usable) });
|
|
193
381
|
if (!result.ok) return { ok: false, error: result.error };
|
|
194
|
-
const { model, effort, reason, risky } = result.data ?? {};
|
|
382
|
+
const { kind, surface, model, effort, reason, risky } = result.data ?? {};
|
|
195
383
|
// The same pairing the schema enforces, checked again: a backend without grammar support can still answer anything.
|
|
196
|
-
const fits = usable.includes(model) && EFFORTS.includes(effort);
|
|
384
|
+
const fits = classify ? KINDS.includes(kind) && SURFACES.includes(surface) : usable.includes(model) && EFFORTS.includes(effort);
|
|
197
385
|
if (!fits || typeof reason !== 'string' || !reason.trim()) return { ok: false, error: `the router answered outside its schema: ${JSON.stringify(result.data)}` };
|
|
198
386
|
// Whether the work touches security, credentials, money, migrations or concurrency: kept, so such work is reviewed. Only a plain yes is one.
|
|
199
|
-
return { ok: true, model, effort, reason: reason.replace(/\s+/g, ' ').trim(), risky: risky === true };
|
|
387
|
+
return { ok: true, kind, surface, model, effort, reason: reason.replace(/\s+/g, ' ').trim(), risky: risky === true };
|
|
200
388
|
}
|
|
201
389
|
|
|
202
|
-
// Routes compared on one scale: by model first,
|
|
390
|
+
// Routes compared on one scale: by model first, weakest to strongest as the job ladder lists them (JOB_MODELS), then by effort.
|
|
203
391
|
// The 'none' an old job ran at, from when haiku took no effort, sits below every effort.
|
|
204
|
-
const modelRank = (model) =>
|
|
392
|
+
const modelRank = (model) => JOB_MODELS.indexOf(model);
|
|
205
393
|
const effortRank = (effort) => EFFORTS.indexOf(effort);
|
|
206
394
|
const compareRoutes = (a, b) => modelRank(a.model) - modelRank(b.model) || effortRank(a.effort) - effortRank(b.effort);
|
|
207
395
|
|
|
396
|
+
/** Whether a retry of `prev` must go above it: the job failed, or drew RETRY_IMPORTANT_THRESHOLD Important findings, on a route Sumo knows. */
|
|
397
|
+
const goesAbove = (prev) => Boolean(prev) && modelRank(prev.model) >= 0 && (prev.status === 'failed' || (prev.important ?? 0) >= RETRY_IMPORTANT_THRESHOLD);
|
|
398
|
+
|
|
208
399
|
/** One step above a route: the next model up that is on, keeping the effort (medium, off an old 'none'); on the top one, one effort up, max at most. */
|
|
209
400
|
function stepAbove({ model, effort }, usable) {
|
|
210
401
|
const next = usable.find((m) => modelRank(m) > modelRank(model));
|
|
@@ -220,13 +411,14 @@ function stepAbove({ model, effort }, usable) {
|
|
|
220
411
|
* - A first attempt starts lower: a worker's never runs on fable while opus is on, and no first attempt runs above high effort.
|
|
221
412
|
* - A scout's first attempt does not run on haiku while sonnet is on: on the ledger, haiku scouts left 3 of 11 unfinished,
|
|
222
413
|
* sonnet and opus ones none of 9 (measured on Haiku 4.5; not yet on 5.5). It runs on sonnet at low effort instead.
|
|
223
|
-
* - A retry of a job that failed, or that drew RETRY_IMPORTANT_THRESHOLD Important findings, runs at least one step above it
|
|
414
|
+
* - A retry of a job that failed, or that drew RETRY_IMPORTANT_THRESHOLD Important findings, runs at least one step above it —
|
|
415
|
+
* unless the scorecard already put it on a route that scores above it (`climbed`): the ladder's next model up may score less.
|
|
224
416
|
* - A reviewer pointed at a job runs on at least that job's model, at medium effort or more.
|
|
225
417
|
* Pure: `retryOf` and `reviews` are the rows (id, model, effort, status, important) of the job retried and the job judged,
|
|
226
418
|
* or nothing; `usable` is the models that are on, cheapest first. An attempt with no route Sumo knows has nothing to go
|
|
227
419
|
* above. `policy` says what changed and why, or is null.
|
|
228
420
|
*/
|
|
229
|
-
export function routePolicy({ agent, routed, retryOf: prev, reviews: target, usable, floor = null }) {
|
|
421
|
+
export function routePolicy({ agent, routed, retryOf: prev, reviews: target, usable, floor = null, climbed = false }) {
|
|
230
422
|
let route = { model: routed.model, effort: routed.effort };
|
|
231
423
|
const why = [];
|
|
232
424
|
// The user's floor before anything else: a job never starts on a model below it while one at or above it is on.
|
|
@@ -250,7 +442,7 @@ export function routePolicy({ agent, routed, retryOf: prev, reviews: target, usa
|
|
|
250
442
|
route.effort = 'high';
|
|
251
443
|
why.push('a first attempt runs at high effort at most');
|
|
252
444
|
}
|
|
253
|
-
} else if (
|
|
445
|
+
} else if (goesAbove(prev) && !climbed) {
|
|
254
446
|
const floor = stepAbove(prev, usable);
|
|
255
447
|
if (compareRoutes(floor, route) > 0) {
|
|
256
448
|
route = floor;
|
|
@@ -287,15 +479,30 @@ const jobRoute = (db, id) => (id === undefined || id === null ? null : (db.prepa
|
|
|
287
479
|
* security, credentials, money, migrations or concurrency — kept with the job, so such work is reviewed once it is done.
|
|
288
480
|
*/
|
|
289
481
|
export async function chooseRoute(db, { agent, project, task, title, retryOf, reviews }) {
|
|
290
|
-
|
|
482
|
+
// With a scorecard the router only sorts the job and code picks the route from it; with none, the router chooses by its rule,
|
|
483
|
+
// among Claude's models, as it always has.
|
|
484
|
+
const card = scorecardFor(db, jobModels(db));
|
|
485
|
+
const routed = await askRouter(db, { agent, project, task, title, retryOf }, { classify: card !== null });
|
|
291
486
|
if (!routed.ok) throw new UsageError(`the router failed: ${routed.error} — no job was created`);
|
|
292
|
-
const
|
|
293
|
-
|
|
487
|
+
const first = retryOf === undefined || retryOf === null;
|
|
488
|
+
const prev = jobRoute(db, retryOf);
|
|
489
|
+
// A retry on the scorecard goes to a route that scores more than the attempt retried, never back to that one.
|
|
490
|
+
const above = card && goesAbove(prev) ? (routeScore(card, prev.model, prev.effort, EFFORTS)?.score ?? null) : null;
|
|
491
|
+
const avoid = [...troubled(db, project.slug), ...(above === null ? [] : [`${prev.model}/${prev.effort}`])];
|
|
492
|
+
const scored = card && pickRoute({ card, usable: jobModels(db), kind: routed.kind, surface: routed.surface, risky: routed.risky, hardens: agent !== 'scout', highest: first ? 'high' : null, avoid, ledger: ledgerCost(db, agent), above });
|
|
493
|
+
const chosen = scored ? { ...routed, model: scored.model, effort: scored.effort } : routed;
|
|
494
|
+
const { model, effort, policy } = routePolicy({ agent, routed: chosen, retryOf: prev, reviews: jobRoute(db, reviews), usable: card ? usableJobModels(db) : usableModels(db), floor: jobFloor(db), climbed: scored?.climbed === true });
|
|
495
|
+
return { model, effort, reason: `router: ${routed.reason}${scored ? `; scorecard: ${scored.why}` : ''}${policy ? `; policy: ${policy}` : ''}`, risky: routed.risky };
|
|
294
496
|
}
|
|
295
497
|
|
|
296
|
-
/** The chat's route for one turn on auto, by the same router: its answer or a refusal, nothing in between. */
|
|
498
|
+
/** The chat's route for one turn on auto, by the same router — sorting the turn when a scorecard is saved, choosing when none is: its answer or a refusal, nothing in between. */
|
|
297
499
|
export async function chooseChatRoute(db, { project, text }) {
|
|
298
|
-
|
|
500
|
+
// The chat runs on Cursor's models too: with a scorecard they are weighed with Claude's; with none, the rule names Claude's alone.
|
|
501
|
+
const card = scorecardFor(db, usableJobModels(db));
|
|
502
|
+
const classify = card !== null;
|
|
503
|
+
const routed = await askFor(db, CHAT_ROUTER_SYSTEM, chatRouterPrompt(db, { project, text }, { classify }), usableModels(db), { classify });
|
|
299
504
|
if (!routed.ok) throw new UsageError(`the router failed: ${routed.error}`);
|
|
300
|
-
|
|
505
|
+
// A turn edits and runs commands as a worker does: its routes are priced by workers' jobs.
|
|
506
|
+
const scored = card && pickRoute({ card, usable: usableJobModels(db), kind: routed.kind, surface: routed.surface, risky: routed.risky, ledger: ledgerCost(db, 'worker') });
|
|
507
|
+
return scored ? { model: scored.model, effort: scored.effort, reason: `${routed.reason}; scorecard: ${scored.why}` } : { model: routed.model, effort: routed.effort, reason: routed.reason };
|
|
301
508
|
}
|