sumo-agents 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +124 -0
- package/README.md +46 -21
- package/package.json +2 -1
- package/prompts/scribe.md +1 -1
- package/src/apply.mjs +16 -12
- package/src/auth.mjs +2 -2
- package/src/card.mjs +3 -2
- package/src/catalog.mjs +100 -13
- package/src/chat.mjs +111 -31
- package/src/check.mjs +17 -15
- package/src/cli.mjs +21 -11
- package/src/cost.mjs +2 -4
- package/src/cursor.mjs +405 -0
- package/src/db.mjs +24 -13
- package/src/dream.mjs +2 -2
- package/src/jobs.mjs +72 -34
- package/src/ledger.mjs +4 -3
- package/src/loop.mjs +157 -77
- package/src/mcp.mjs +7 -9
- package/src/memeval.mjs +2 -3
- package/src/memory.mjs +5 -4
- package/src/model.mjs +5 -0
- package/src/page.mjs +1 -0
- package/src/prime.mjs +11 -7
- package/src/rebuild.mjs +8 -3
- package/src/redact.mjs +13 -1
- package/src/route.mjs +263 -56
- package/src/sandbox.mjs +86 -20
- package/src/scan.mjs +4 -2
- package/src/scorecard.mjs +419 -0
- package/src/scribe.mjs +82 -38
- package/src/sessions.mjs +29 -21
- package/src/settings.mjs +114 -37
- package/src/setup.mjs +33 -14
- package/src/spool.mjs +68 -43
- package/src/tools.mjs +220 -98
- package/src/ui.mjs +14 -5
- package/src/verify.mjs +116 -15
- package/src/worktree.mjs +93 -9
package/src/sandbox.mjs
CHANGED
|
@@ -7,8 +7,8 @@ import { paths } from './paths.mjs';
|
|
|
7
7
|
/**
|
|
8
8
|
* The OS sandbox around the model's shell: macOS Seatbelt, the way Codex and Claude Code wrap their commands. Reads and
|
|
9
9
|
* the network stay open — git over ssh reads ~/.ssh, a smoke test calls out, and the user chose to give the shell their
|
|
10
|
-
* environment — while writes are held to the project, its git directory, the temporary folders
|
|
11
|
-
*
|
|
10
|
+
* environment — while writes are held to the project, its git directory, the temporary folders and the tool caches —
|
|
11
|
+
* never Sumo's own home — and never reach git's hooks or config, which run later outside it. The user can close the network
|
|
12
12
|
* past this machine (`shell.network off`). The guard refuses the commands it knows; this stops the ones it does not,
|
|
13
13
|
* whatever they are spelled as.
|
|
14
14
|
* `sandbox-exec` execs into the shell in place, so the shell keeps its pid and its process group, and a stop still reaches it.
|
|
@@ -38,6 +38,24 @@ const real = (path) => {
|
|
|
38
38
|
}
|
|
39
39
|
};
|
|
40
40
|
|
|
41
|
+
/** git's commands that look into a submodule's working tree — run in it, under its own config — unless told not to. */
|
|
42
|
+
const LOOK_INTO_SUBMODULES = new Set(['status', 'diff', 'diff-index', 'diff-files']);
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* A git command as sumo runs it for itself, outside the sandbox, in a folder the model's shell can write: nothing the
|
|
46
|
+
* folder's git files name is run — no fsmonitor, no hook — and no submodule is gone into, whose config the shell could
|
|
47
|
+
* have planted (a gitlink in the index and a repository beside it are all it takes). `.gitmodules` can set a
|
|
48
|
+
* submodule's `ignore` over any config, so the commands that would look into one are told on their own command line,
|
|
49
|
+
* which nothing in the folder overrides; the fsmonitor setting reaches the git a submodule would run all the same.
|
|
50
|
+
* `git config` runs nothing, and is asked what the folder's config says: it is left to say it, hooks path and all.
|
|
51
|
+
*/
|
|
52
|
+
export function ownGit(args) {
|
|
53
|
+
const [command, ...rest] = args;
|
|
54
|
+
if (command === 'config') return args;
|
|
55
|
+
const blind = LOOK_INTO_SUBMODULES.has(command) && !rest.includes('--no-index') ? ['--ignore-submodules=dirty'] : [];
|
|
56
|
+
return ['-c', 'core.fsmonitor=false', '-c', 'core.hooksPath=/dev/null', '-c', 'submodule.recurse=false', '-c', 'diff.ignoreSubmodules=dirty', command, ...blind, ...rest];
|
|
57
|
+
}
|
|
58
|
+
|
|
41
59
|
/** Each root's git common directory, asked of git once: a root does not turn into a worktree of another repository halfway through. */
|
|
42
60
|
const commonDirs = new Map();
|
|
43
61
|
|
|
@@ -46,7 +64,7 @@ function gitCommonDir(root) {
|
|
|
46
64
|
if (!commonDirs.has(root)) {
|
|
47
65
|
let dir = null;
|
|
48
66
|
try {
|
|
49
|
-
const said = execFileSync('git', ['rev-parse', '--git-common-dir'], { cwd: root, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: 5_000 }).trim();
|
|
67
|
+
const said = execFileSync('git', ownGit(['rev-parse', '--git-common-dir']), { cwd: root, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: 5_000 }).trim();
|
|
50
68
|
dir = said ? real(resolve(root, said)) : null;
|
|
51
69
|
} catch {
|
|
52
70
|
// Not in a repository, or no git: there is nothing outside the root to write history to.
|
|
@@ -67,7 +85,7 @@ function hooksPathOf(root) {
|
|
|
67
85
|
if (!hooksPaths.has(root)) {
|
|
68
86
|
let dir = null;
|
|
69
87
|
try {
|
|
70
|
-
const ask = (...args) => execFileSync('git', args, { cwd: root, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: 5_000 }).trim();
|
|
88
|
+
const ask = (...args) => execFileSync('git', ownGit(args), { cwd: root, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: 5_000 }).trim();
|
|
71
89
|
const said = ask('config', '--type=path', '--get', 'core.hooksPath');
|
|
72
90
|
if (said) {
|
|
73
91
|
const path = resolve(ask('rev-parse', '--show-toplevel') || root, said);
|
|
@@ -96,14 +114,16 @@ function darwinUserCache() {
|
|
|
96
114
|
|
|
97
115
|
/**
|
|
98
116
|
* The places a sandboxed command may write, by their real paths — the sandbox sees through a link — each once: every
|
|
99
|
-
* root and the git directory it commits to, the temporary folders,
|
|
100
|
-
*
|
|
101
|
-
*
|
|
117
|
+
* root given and the git directory it commits to, the temporary folders, the per-user cache macOS gives compilers, and
|
|
118
|
+
* the tool caches that exist. A place that does not exist is left out. Never Sumo's home: what is in it is read later by
|
|
119
|
+
* sumo, outside the sandbox — an /undo turn it restores, a backup it puts back, another job's inbox read as the user's
|
|
120
|
+
* words — and the sumo commands the shell runs, run as sumo outside it, write there themselves. A root inside the home
|
|
121
|
+
* (a scratch chore, a job's worktree) is that root's.
|
|
102
122
|
*/
|
|
103
|
-
export function writablePlaces(roots, { home = homedir(), tmp = tmpdir(),
|
|
123
|
+
export function writablePlaces(roots, { home = homedir(), tmp = tmpdir(), userCache = darwinUserCache(), opened = [] } = {}) {
|
|
104
124
|
const places = [];
|
|
105
125
|
for (const root of roots.map(real).filter(Boolean)) places.push(root, gitCommonDir(root));
|
|
106
|
-
places.push(real(tmp), real('/private/tmp'),
|
|
126
|
+
places.push(real(tmp), real('/private/tmp'), userCache && real(userCache), ...TOOL_CACHES.map((cache) => real(join(home, cache))));
|
|
107
127
|
// What the user opened for the session at the editor's question — a file, or a directory with everything under it — the shell may write too.
|
|
108
128
|
places.push(...[...opened].map(real));
|
|
109
129
|
return [...new Set(places.filter(Boolean))];
|
|
@@ -117,7 +137,7 @@ export function writablePlaces(roots, { home = homedir(), tmp = tmpdir(), sumo =
|
|
|
117
137
|
const GUARDED = {
|
|
118
138
|
'your shell reads it when it starts': ['.zshrc', '.zshenv', '.zprofile', '.zlogin', '.bashrc', '.bash_profile', '.bash_login', '.profile', '.envrc'],
|
|
119
139
|
// Husky's hooks are the directory git's config names (`.husky/_`), kept with git's own; the rest of `.husky` is the project's.
|
|
120
|
-
'a tool reads it before it runs': ['.npmrc', '.yarnrc', '.yarnrc.yml', 'bunfig.toml', '.pre-commit-config.yaml', 'lefthook.yml', '.mcp.json', '.claude', '.codex', '.agents'],
|
|
140
|
+
'a tool reads it before it runs': ['.npmrc', '.yarnrc', '.yarnrc.yml', 'bunfig.toml', '.pre-commit-config.yaml', 'lefthook.yml', '.mcp.json', '.claude', '.codex', '.agents', '.cursor'],
|
|
121
141
|
'your editor reads it': ['.vscode', '.idea', '.devcontainer'],
|
|
122
142
|
// Keys and logins other programs trust: a line added to authorized_keys or a profile in .aws outlives every session.
|
|
123
143
|
'it holds credentials other programs trust': ['.ssh', '.aws', '.gnupg', '.kube', '.docker', '.netrc', '.git-credentials', '.pgpass'],
|
|
@@ -138,9 +158,13 @@ const HOME_WHY = [...GUARDED_WHY, ...whyOf(HOME_GUARDED)];
|
|
|
138
158
|
|
|
139
159
|
/**
|
|
140
160
|
* What sumo keeps for itself, which only sumo writes: its memory and settings, its MCP servers, its own program and model —
|
|
141
|
-
*
|
|
161
|
+
* the spool, whose lines are replayed into the journal as what the user said; the memory pass's locks, by name, there or
|
|
162
|
+
* not (one naming a process that never ends would stop every pass); its logs, which hold the saved sessions (resumed as
|
|
163
|
+
* the conversation itself) and each turn's /undo (whose files sumo writes back); the backups a restore puts back; and
|
|
164
|
+
* the chat's history of what was typed. Nothing sandboxed writes them: the passes and the chat run as sumo. The home is
|
|
165
|
+
* not the shell's to write at all (`writablePlaces`); these hold where it lies in a place that is — a home under /tmp.
|
|
142
166
|
*/
|
|
143
|
-
const SUMO_PRIVATE = ['memory.db', 'memory.db-wal', 'memory.db-shm', 'mcp.json', 'bin', 'models', 'eval.env', 'spool'];
|
|
167
|
+
const SUMO_PRIVATE = ['memory.db', 'memory.db-wal', 'memory.db-shm', 'mcp.json', 'bin', 'models', 'eval.env', 'spool', 'scribe.lock', 'dream.lock', 'logs', 'backups', 'history'];
|
|
144
168
|
|
|
145
169
|
/** The directories on the PATH, absolute: a program put in one runs whenever its name is typed. */
|
|
146
170
|
const pathDirs = (env = process.env) => [...new Set(String(env.PATH ?? '').split(':').filter((d) => d.startsWith('/')).map((d) => resolve(d)))];
|
|
@@ -173,14 +197,23 @@ export function guardedWhy(path, { roots = [], home = homedir(), sumo = paths().
|
|
|
173
197
|
}
|
|
174
198
|
|
|
175
199
|
/**
|
|
176
|
-
*
|
|
200
|
+
* Guarded names directly under a folder, by path there or not — a `.envrc` or `.claude/settings.json` the shell made
|
|
201
|
+
* would be read later all the same — and, for one that is a link, where it leads too.
|
|
202
|
+
*/
|
|
203
|
+
const guardedUnder = (dir, names) => {
|
|
204
|
+
const base = real(dir);
|
|
205
|
+
return base ? names.flatMap((name) => [join(base, name), real(join(base, name))]) : [];
|
|
206
|
+
};
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* What the shell may not write inside the places it may: the guarded names in each root and the home, there or not;
|
|
177
210
|
* sumo's own files by name, there or not (`mcp.json` made by the shell would be read as servers allowed to run); and
|
|
178
211
|
* the PATH's folders outside the project. Kept read-only as git's hooks are.
|
|
179
212
|
*/
|
|
180
213
|
export function guardedPlaces(roots, { home = homedir(), sumo = paths().home, env = process.env } = {}) {
|
|
181
214
|
const places = [];
|
|
182
|
-
for (const dir of roots) places.push(...Object.values(GUARDED).flat()
|
|
183
|
-
places.push(...[...Object.values(GUARDED), ...Object.values(HOME_GUARDED)].flat()
|
|
215
|
+
for (const dir of roots) places.push(...guardedUnder(dir, Object.values(GUARDED).flat()));
|
|
216
|
+
places.push(...guardedUnder(home, [...Object.values(GUARDED), ...Object.values(HOME_GUARDED)].flat()));
|
|
184
217
|
const own = real(sumo) ?? sumo;
|
|
185
218
|
places.push(...SUMO_PRIVATE.map((name) => join(own, name)));
|
|
186
219
|
// A PATH folder in a project is the project's; one above a project would make all of it read-only.
|
|
@@ -219,6 +252,35 @@ export function readOnlyPlaces(roots) {
|
|
|
219
252
|
return [...new Set(places)];
|
|
220
253
|
}
|
|
221
254
|
|
|
255
|
+
/** A path as a pattern that matches it alone. */
|
|
256
|
+
const literally = (path) => path.replace(/[.*+?^${}()|[\]\\]/g, (c) => `\\${c}`);
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* What stays read-only by its shape rather than by a path that is there (seatbelt's regex matches case as the disk does):
|
|
260
|
+
* a repository's config and hooks below each root — one the shell made or cloned is as much git's to run later, by the
|
|
261
|
+
* user's `git status` going into a submodule or an editor scanning the folder, as the root's own — and the `.git` there
|
|
262
|
+
* itself, so neither a `.git` file pointing elsewhere nor a whole `.git` moved in brings its own; the config and hooks
|
|
263
|
+
* of the root repository's submodules and worktrees; and the copies of the database a migration leaves, which a restore
|
|
264
|
+
* puts back. A root's own `git init` still makes its own. Not caught: a folder moved in whole from a place the shell may
|
|
265
|
+
* write, with a repository inside it — sumo's own git never runs what such a one names (`ownGit`).
|
|
266
|
+
*/
|
|
267
|
+
export function readOnlyPatterns(roots, { sumo = paths().home } = {}) {
|
|
268
|
+
const own = '(config|config\\.worktree|hooks(/.*)?)';
|
|
269
|
+
const patterns = [];
|
|
270
|
+
for (const root of roots.map(real).filter(Boolean)) {
|
|
271
|
+
const r = literally(root);
|
|
272
|
+
// The root's own repository too, made or not: `git init --template` in a root with none writes a config and hooks
|
|
273
|
+
// git runs later outside the sandbox — your prompt's `git status`, an editor — the same as a planted nested one.
|
|
274
|
+
patterns.push(`^${r}/.+/\\.git$`, `^${r}/.+/\\.git/(.+/)?${own}$`, `^${r}/\\.git/${own}$`);
|
|
275
|
+
const gitDir = real(join(root, '.git'));
|
|
276
|
+
for (const dir of new Set([gitDir && isDirectory(gitDir) ? gitDir : join(root, '.git'), gitCommonDir(root)].filter(Boolean))) {
|
|
277
|
+
patterns.push(`^${literally(dir)}/(modules|worktrees)/(.+/)?${own}$`);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
patterns.push(`^${literally(real(sumo) ?? sumo)}/memory\\.db\\.pre-v[0-9]+\\.bak$`);
|
|
281
|
+
return [...new Set(patterns)];
|
|
282
|
+
}
|
|
283
|
+
|
|
222
284
|
/** Every directory above a path, up to and without the root of the disk. */
|
|
223
285
|
function ancestors(path) {
|
|
224
286
|
const above = [];
|
|
@@ -229,25 +291,29 @@ function ancestors(path) {
|
|
|
229
291
|
/**
|
|
230
292
|
* What a command runs under: sandbox-exec with a profile that allows everything but writing, then writing to each
|
|
231
293
|
* place — given as a parameter, so no path is ever spelled inside the profile — but never to what is kept read-only in
|
|
232
|
-
* them, carved out with require-not as Codex does
|
|
294
|
+
* them, carved out with require-not as Codex does, nor to a path of a shape kept read-only (`patterns`, regexes, given as
|
|
295
|
+
* parameters too); and to the devices every shell writes to. No directory above a path
|
|
233
296
|
* kept read-only can be moved or removed: renamed aside, what is under it would be out of the carve-out, and could be
|
|
234
297
|
* written and moved back. With the network off, nothing past this machine is reached; loopback — a dev server, a
|
|
235
298
|
* test's own — and local sockets stay open. The shell and its command follow it.
|
|
236
299
|
*/
|
|
237
|
-
export function sandboxWrap(writable, { readOnly = [], network = true } = {}) {
|
|
300
|
+
export function sandboxWrap(writable, { readOnly = [], patterns = [], network = true } = {}) {
|
|
238
301
|
const places = writable.map((_, i) => `(subpath (param "W${i}"))`).join(' ');
|
|
239
|
-
const kept =
|
|
302
|
+
const kept = [
|
|
303
|
+
...readOnly.map((_, i) => `(require-not (literal (param "R${i}"))) (require-not (subpath (param "R${i}")))`),
|
|
304
|
+
...patterns.map((_, i) => `(require-not (regex (param "P${i}")))`),
|
|
305
|
+
].join(' ');
|
|
240
306
|
const above = [...new Set(readOnly.flatMap(ancestors))];
|
|
241
307
|
const rules = [
|
|
242
308
|
'(version 1)',
|
|
243
309
|
'(allow default)',
|
|
244
310
|
'(deny file-write*)',
|
|
245
|
-
`(allow file-write* ${
|
|
311
|
+
`(allow file-write* ${kept === '' ? places : `(require-all (require-any ${places}) ${kept})`} (literal "/dev/null") (regex #"^/dev/(tty|fd/|zero|dtracehelper)"))`,
|
|
246
312
|
];
|
|
247
313
|
if (!network) rules.push('(deny network-outbound (remote ip))', '(allow network-outbound (remote ip "localhost:*"))');
|
|
248
314
|
// Last, so no allowance after it opens the unlink a rename is made of.
|
|
249
315
|
if (above.length > 0) rules.push(`(deny file-write-unlink (require-all (vnode-type DIRECTORY) (require-any ${above.map((_, i) => `(literal (param "A${i}"))`).join(' ')})))`);
|
|
250
|
-
const params = [...writable.map((place, i) => `-DW${i}=${place}`), ...readOnly.map((place, i) => `-DR${i}=${place}`), ...above.map((place, i) => `-DA${i}=${place}`)];
|
|
316
|
+
const params = [...writable.map((place, i) => `-DW${i}=${place}`), ...readOnly.map((place, i) => `-DR${i}=${place}`), ...patterns.map((pattern, i) => `-DP${i}=${pattern}`), ...above.map((place, i) => `-DA${i}=${place}`)];
|
|
251
317
|
return [SANDBOX_EXEC, '-p', rules.join('\n'), ...params, '--'];
|
|
252
318
|
}
|
|
253
319
|
|
package/src/scan.mjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { execFileSync } from 'node:child_process';
|
|
2
2
|
import { closeSync, constants, existsSync, fstatSync, openSync, readdirSync, readFileSync, realpathSync, statSync } from 'node:fs';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
|
+
import { ownGit } from './sandbox.mjs';
|
|
4
5
|
|
|
5
6
|
/**
|
|
6
7
|
* Reads what a repository says about itself — deterministically, with no model.
|
|
@@ -55,7 +56,7 @@ const readJson = (root, name) => {
|
|
|
55
56
|
|
|
56
57
|
export function gitRemote(root) {
|
|
57
58
|
try {
|
|
58
|
-
return execFileSync('git', ['-C', root, 'remote', 'get-url', 'origin'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim() || null;
|
|
59
|
+
return execFileSync('git', ['-C', root, ...ownGit(['remote', 'get-url', 'origin'])], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim() || null;
|
|
59
60
|
} catch {
|
|
60
61
|
return null;
|
|
61
62
|
}
|
|
@@ -131,7 +132,8 @@ function projectCommands(root, pkg = readJson(root, 'package.json')) {
|
|
|
131
132
|
}
|
|
132
133
|
if (found.length === 0) {
|
|
133
134
|
if (existsSync(join(root, 'go.mod'))) found.push({ name: 'test', command: 'go test ./...' });
|
|
134
|
-
|
|
135
|
+
// Every test target run, not stopped at the first that fails: what a red one stops is never run, before the work or after it.
|
|
136
|
+
if (existsSync(join(root, 'Cargo.toml'))) found.push({ name: 'test', command: 'cargo test --no-fail-fast' });
|
|
135
137
|
if (/\bpytest\b/.test(read(root, 'pyproject.toml') ?? '')) found.push({ name: 'test', command: 'pytest' });
|
|
136
138
|
}
|
|
137
139
|
return found;
|
|
@@ -0,0 +1,419 @@
|
|
|
1
|
+
import { execFileSync, spawnSync } from 'node:child_process';
|
|
2
|
+
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { ALL_MODELS, modelId, NAME_WIDTH } from './catalog.mjs';
|
|
6
|
+
import { getMeta, setMeta } from './db.mjs';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* How good each model is at each effort, from benchmarks other people publish: fetched once, by Discover (and the first
|
|
10
|
+
* `sumo setup`), and saved; routing reads what was saved and fetches nothing. Three sources, none needing a key, each
|
|
11
|
+
* scoring on its own scale — so a score is never averaged as it is: within each source it becomes a place among Sumo's
|
|
12
|
+
* models, 0 the worst and 1 the best, and a route's score is the mean of its places in the sources that list it.
|
|
13
|
+
* - CursorBench, as Epoch AI collects it (CC-BY): coding-agent tasks passed, and the cost of one, for every model and effort.
|
|
14
|
+
* - LMArena's agent leaderboard (CC-BY-4.0): people's votes in agent sessions, per model and effort.
|
|
15
|
+
* - Artificial Analysis's Intelligence Index, as modelgrep serves it: one general score per model.
|
|
16
|
+
* Code reads every number; no model reads a page here.
|
|
17
|
+
*/
|
|
18
|
+
export const SOURCES = [
|
|
19
|
+
{ id: 'cursorbench', name: 'CursorBench, via Epoch AI', url: 'https://epoch.ai/data/benchmark_data.zip' },
|
|
20
|
+
{ id: 'arena', name: 'LMArena agent leaderboard', url: 'https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset' },
|
|
21
|
+
{ id: 'intelligence', name: 'Artificial Analysis Intelligence Index, via modelgrep', url: 'https://modelgrep.com/api' },
|
|
22
|
+
];
|
|
23
|
+
|
|
24
|
+
const KEY = 'scorecard';
|
|
25
|
+
const TIMEOUT_MS = 30_000;
|
|
26
|
+
const ARENA_ROWS = 'https://datasets-server.huggingface.co/rows?dataset=lmarena-ai/leaderboard-dataset&config=agent&split=latest';
|
|
27
|
+
const MODELGREP = 'https://modelgrep.com/api/v1/models';
|
|
28
|
+
/** What the sources call an effort, as Sumo calls it; a level Sumo has no word for (none, minimal, unknown) is left out. */
|
|
29
|
+
const EFFORT_WORDS = { low: 'low', medium: 'medium', high: 'high', xhigh: 'xhigh', 'extra high': 'xhigh', max: 'max' };
|
|
30
|
+
/** A model's own row, when a source scores the model and not one effort of it. */
|
|
31
|
+
const ANY = '*';
|
|
32
|
+
|
|
33
|
+
/** A model's name as the sources and the API spell it, the same way: no maker, no punctuation, lower case — `Claude Haiku 5.5` and `anthropic/claude-haiku-5.5` are both `claude-haiku-5-5`. */
|
|
34
|
+
const plain = (name) => String(name).toLowerCase().replace(/^[^/]*\//, '').replace(/[\s._]+/g, '-').replace(/^-+|-+$/g, '');
|
|
35
|
+
|
|
36
|
+
/** The Sumo model a source's name means, or null for one Sumo does not run. */
|
|
37
|
+
function sumoModel(name, models) {
|
|
38
|
+
const wanted = plain(name);
|
|
39
|
+
return models.find((m) => plain(modelId(m)) === wanted) ?? null;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** The effort a source's word means: a level Sumo has, `*` for none named, null for one Sumo has no word for. */
|
|
43
|
+
function effortOf(word) {
|
|
44
|
+
if (word === undefined || word === null || String(word).trim() === '') return ANY;
|
|
45
|
+
return EFFORT_WORDS[String(word).trim().toLowerCase()] ?? null;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* One entry per model and effort a source scores, keeping the higher score where it lists one twice. `tally`, when given, counts
|
|
50
|
+
* what was left out and why: rows of models Sumo does not run, at an effort it has no word for, with no score, or listed twice.
|
|
51
|
+
*/
|
|
52
|
+
function entriesOf(rows, models, tally = null) {
|
|
53
|
+
const kept = new Map();
|
|
54
|
+
const count = (why) => tally && (tally[why] = (tally[why] ?? 0) + 1);
|
|
55
|
+
for (const { name, effort: word, score, cost } of rows) {
|
|
56
|
+
const model = sumoModel(name, models);
|
|
57
|
+
const effort = effortOf(word);
|
|
58
|
+
if (!model) count('model');
|
|
59
|
+
else if (effort === null) count('effort');
|
|
60
|
+
else if (!Number.isFinite(score)) count('score');
|
|
61
|
+
else {
|
|
62
|
+
const key = `${model}/${effort}`;
|
|
63
|
+
if (kept.has(key)) count('twice');
|
|
64
|
+
if (!kept.has(key) || kept.get(key).score < score) kept.set(key, { model, effort, score, cost: Number.isFinite(cost) ? cost : null });
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
if (tally) tally.rows = rows.length;
|
|
68
|
+
return [...kept.values()];
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** A CSV file's rows as objects by header: quoted fields may hold commas, newlines and doubled quotes. */
|
|
72
|
+
export function parseCsv(text) {
|
|
73
|
+
const rows = [];
|
|
74
|
+
let row = [];
|
|
75
|
+
let field = '';
|
|
76
|
+
let quoted = false;
|
|
77
|
+
for (let i = 0; i < text.length; i += 1) {
|
|
78
|
+
const c = text[i];
|
|
79
|
+
if (quoted) {
|
|
80
|
+
if (c === '"' && text[i + 1] === '"') {
|
|
81
|
+
field += '"';
|
|
82
|
+
i += 1;
|
|
83
|
+
} else if (c === '"') quoted = false;
|
|
84
|
+
else field += c;
|
|
85
|
+
} else if (c === '"') quoted = true;
|
|
86
|
+
else if (c === ',') {
|
|
87
|
+
row.push(field);
|
|
88
|
+
field = '';
|
|
89
|
+
} else if (c === '\n' || c === '\r') {
|
|
90
|
+
if (c === '\r' && text[i + 1] === '\n') i += 1;
|
|
91
|
+
row.push(field);
|
|
92
|
+
rows.push(row);
|
|
93
|
+
row = [];
|
|
94
|
+
field = '';
|
|
95
|
+
} else field += c;
|
|
96
|
+
}
|
|
97
|
+
if (field !== '' || row.length > 0) rows.push([...row, field]);
|
|
98
|
+
const [header = [], ...body] = rows.filter((r) => r.some((f) => f !== ''));
|
|
99
|
+
return body.map((r) => Object.fromEntries(header.map((h, i) => [h, r[i] ?? ''])));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** CursorBench's rows: `claude-opus-5-5_high` is a model and its effort; a name with no effort after it scores the model. */
|
|
103
|
+
export function fromCursorBench(csv, models = ALL_MODELS, tally = null) {
|
|
104
|
+
const rows = parseCsv(csv).map((r) => {
|
|
105
|
+
const version = r['Model version'] ?? '';
|
|
106
|
+
const cut = version.lastIndexOf('_');
|
|
107
|
+
const [name, effort] = cut > 0 ? [version.slice(0, cut), version.slice(cut + 1)] : [version, undefined];
|
|
108
|
+
return { name, effort, score: Number.parseFloat(r.Score), cost: Number.parseFloat(r['Cost per task']) };
|
|
109
|
+
});
|
|
110
|
+
return entriesOf(rows, models, tally);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** LMArena's rows: `Claude Opus 5.5 (High)` is a model and its effort; only the overall board counts. */
|
|
114
|
+
export function fromArena(rows, models = ALL_MODELS, tally = null) {
|
|
115
|
+
const read = rows
|
|
116
|
+
.filter((r) => (r.category ?? 'overall') === 'overall')
|
|
117
|
+
.map((r) => {
|
|
118
|
+
const named = /^(.*?)\s*\(([^)]*)\)\s*$/.exec(r.model_name ?? '');
|
|
119
|
+
return { name: named ? named[1] : r.model_name, effort: named ? named[2] : undefined, score: r.score, cost: r.cost_per_task_p50 };
|
|
120
|
+
});
|
|
121
|
+
return entriesOf(read, models, tally);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** modelgrep's rows: one Intelligence Index per model, at no effort in particular. */
|
|
125
|
+
export function fromIntelligence(rows, models = ALL_MODELS, tally = null) {
|
|
126
|
+
return entriesOf(rows.map((r) => ({ name: r.id, score: r.benchmarks?.artificial_analysis?.intelligence })), models, tally);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** The models a source's entries score, each once. */
|
|
130
|
+
const modelsIn = (entries) => [...new Set(entries.map((e) => e.model))];
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Each entry's place among the others in its source: 0 the lowest score, 1 the highest, ties sharing a place. A source that
|
|
134
|
+
* scores fewer than two of Sumo's models says nothing: one model's efforts placed against each other would put its lower one at
|
|
135
|
+
* the bottom of Sumo's models, below models the source never ranked.
|
|
136
|
+
*/
|
|
137
|
+
function placed(entries) {
|
|
138
|
+
if (modelsIn(entries).length < 2) return [];
|
|
139
|
+
const sorted = entries.map((e) => e.score).sort((a, b) => a - b);
|
|
140
|
+
const rank = (score) => {
|
|
141
|
+
const first = sorted.indexOf(score);
|
|
142
|
+
const last = sorted.lastIndexOf(score);
|
|
143
|
+
return (first + last) / 2 / (sorted.length - 1);
|
|
144
|
+
};
|
|
145
|
+
return entries.map((e) => ({ ...e, place: rank(e.score) }));
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** Bytes as a person reads them. */
|
|
149
|
+
const sized = (n) => (n >= 1_000_000 ? `${(n / 1_000_000).toFixed(1)} MB` : n >= 1000 ? `${(n / 1000).toFixed(1)} KB` : `${n} B`);
|
|
150
|
+
/** How long a source that failed is given before it is tried once more: a dropped connection is often back by then. */
|
|
151
|
+
const RETRY_MS = 2000;
|
|
152
|
+
/** The most one answer may send — the archive is 2.3 MB today — and the longest one source may take, every page of it, each try. */
|
|
153
|
+
const MAX_BYTES = 32 * 1024 * 1024;
|
|
154
|
+
const SOURCE_MS = 90_000;
|
|
155
|
+
/**
|
|
156
|
+
* Why a request failed, in words: a timeout is said as one, not as an abort — the source's own deadline (`limits`) apart from
|
|
157
|
+
* one request's — and Node's bare "fetch failed" with what was under it: the connection reset, the name not found, the
|
|
158
|
+
* certificate refused.
|
|
159
|
+
*/
|
|
160
|
+
function failedBecause(cause, limits) {
|
|
161
|
+
if (limits?.deadline?.aborted) return `not read in ${limits.sourceMs / 1000} s, the most a source is given`;
|
|
162
|
+
if (cause?.name === 'TimeoutError') return `no answer in ${TIMEOUT_MS / 1000} s`;
|
|
163
|
+
const under = cause?.cause;
|
|
164
|
+
const detail = under && (under.code || under.message) ? ` (${[under.code, under.code ? null : under.message].filter(Boolean).join(': ')})` : '';
|
|
165
|
+
return `${String(cause?.message ?? cause)}${detail}`.slice(0, 200);
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* One GET, said as it goes (`say`): the address, then the status, the size and the time. The body comes back as bytes, read
|
|
170
|
+
* no further than `limits.maxBytes`, and the request ends at `limits.deadline`, the source's, if that comes first.
|
|
171
|
+
*/
|
|
172
|
+
async function get(fetchImpl, url, say, limits) {
|
|
173
|
+
say(`GET ${url}`);
|
|
174
|
+
const started = Date.now();
|
|
175
|
+
limits.deadline.throwIfAborted();
|
|
176
|
+
const res = await fetchImpl(url, { signal: AbortSignal.any([AbortSignal.timeout(TIMEOUT_MS), limits.deadline]), headers: { 'user-agent': 'sumo-agents' } });
|
|
177
|
+
const tooBig = () => new Error(`sent more than ${sized(limits.maxBytes)}, the most a source may send`);
|
|
178
|
+
if (Number(res.headers.get('content-length')) > limits.maxBytes) throw tooBig();
|
|
179
|
+
const chunks = [];
|
|
180
|
+
let length = 0;
|
|
181
|
+
for await (const chunk of res.body ?? []) {
|
|
182
|
+
length += chunk.length;
|
|
183
|
+
if (length > limits.maxBytes) throw tooBig();
|
|
184
|
+
chunks.push(chunk);
|
|
185
|
+
}
|
|
186
|
+
const body = Buffer.concat(chunks);
|
|
187
|
+
say(` ${res.status} · ${sized(body.length)} · ${((Date.now() - started) / 1000).toFixed(2)} s`);
|
|
188
|
+
if (!res.ok) throw new Error(`answered ${res.status}`);
|
|
189
|
+
return body;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/** Every page of a listing that says how many rows it has, `limit` at a time, until the source's deadline. */
|
|
193
|
+
async function allPages(fetchImpl, url, say, limits, { limit, rowsOf, totalOf, size = 'length' }) {
|
|
194
|
+
const rows = [];
|
|
195
|
+
for (let at = 0; at < 2000; at += limit) {
|
|
196
|
+
const page = JSON.parse(await get(fetchImpl, `${url}${url.includes('?') ? '&' : '?'}offset=${at}&${size}=${limit}`, say, limits));
|
|
197
|
+
rows.push(...rowsOf(page));
|
|
198
|
+
if (rows.length >= totalOf(page) || rowsOf(page).length === 0) break;
|
|
199
|
+
}
|
|
200
|
+
say(` ${rows.length} rows`);
|
|
201
|
+
return rows;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/** The one file Sumo reads out of Epoch's archive, with the unzip macOS ships: no library to unpack it. */
|
|
205
|
+
function cursorBenchCsv(bytes) {
|
|
206
|
+
const dir = mkdtempSync(join(tmpdir(), 'sumo-scorecard-'));
|
|
207
|
+
try {
|
|
208
|
+
const file = join(dir, 'benchmark_data.zip');
|
|
209
|
+
writeFileSync(file, bytes);
|
|
210
|
+
return execFileSync('/usr/bin/unzip', ['-p', file, 'cursorbench_external.csv'], { encoding: 'utf8', maxBuffer: 16 * 1024 * 1024 });
|
|
211
|
+
} finally {
|
|
212
|
+
rmSync(dir, { recursive: true, force: true });
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/** How each source is read: its requests, said as they go and held to `limits`, then its rows read for Sumo's models. */
|
|
217
|
+
const READERS = {
|
|
218
|
+
cursorbench: async (fetchImpl, say, tally, limits) => {
|
|
219
|
+
const csv = cursorBenchCsv(await get(fetchImpl, SOURCES[0].url, say, limits));
|
|
220
|
+
say(' unzipped cursorbench_external.csv');
|
|
221
|
+
return fromCursorBench(csv, ALL_MODELS, tally);
|
|
222
|
+
},
|
|
223
|
+
arena: async (fetchImpl, say, tally, limits) =>
|
|
224
|
+
fromArena(await allPages(fetchImpl, ARENA_ROWS, say, limits, { limit: 100, rowsOf: (p) => (p.rows ?? []).map((r) => r.row), totalOf: (p) => p.num_rows_total ?? 0 }), ALL_MODELS, tally),
|
|
225
|
+
intelligence: async (fetchImpl, say, tally, limits) =>
|
|
226
|
+
fromIntelligence(await allPages(fetchImpl, MODELGREP, say, limits, { limit: 200, size: 'limit', rowsOf: (p) => p.data ?? [], totalOf: (p) => p.meta?.total ?? 0 }), ALL_MODELS, tally),
|
|
227
|
+
};
|
|
228
|
+
|
|
229
|
+
/** What a source's rows came to: how many were kept, for which models, and what was left out and why. */
|
|
230
|
+
function keptLines(entries, tally) {
|
|
231
|
+
const per = ALL_MODELS.map((m) => [m, entries.filter((e) => e.model === m).length]).filter(([, n]) => n > 0);
|
|
232
|
+
const lines = [`kept ${entries.length} of ${tally.rows} rows${per.length > 0 ? `: ${per.map(([m, n]) => `${m} ${n}`).join(', ')}` : ''}`];
|
|
233
|
+
const why = { model: "of models Sumo doesn't run", effort: 'at an effort Sumo has no word for', score: 'with no score', twice: 'listed twice (the better kept)' };
|
|
234
|
+
const left = Object.entries(why).filter(([k]) => tally[k] > 0).map(([k, said]) => `${tally[k]} ${said}`);
|
|
235
|
+
if (left.length > 0) lines.push(`left out ${left.join(', ')}`);
|
|
236
|
+
return lines;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/** Why a source's entries place nothing (see `placed`), or null when they place Sumo's models. */
|
|
240
|
+
function unplaced(entries) {
|
|
241
|
+
const models = modelsIn(entries);
|
|
242
|
+
if (models.length === 0) return "none of Sumo's models is listed";
|
|
243
|
+
return models.length === 1 ? `only ${models[0]} of Sumo's models is listed: nothing to rank it against` : null;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/** Where a source placed Sumo's routes: its best and its worst, by score. */
|
|
247
|
+
function placeLine(entries) {
|
|
248
|
+
const none = unplaced(entries);
|
|
249
|
+
if (none) return none;
|
|
250
|
+
const sorted = [...entries].sort((a, b) => b.score - a.score);
|
|
251
|
+
const route = (e) => `${e.model}${e.effort === ANY ? '' : `/${e.effort}`}`;
|
|
252
|
+
// A source's own score, to three places: enough to see the order, not the noise of a float.
|
|
253
|
+
const own = (e) => Number(e.score.toFixed(3));
|
|
254
|
+
return `places: best ${route(sorted[0])} (${own(sorted[0])}) 1.00 · worst ${route(sorted.at(-1))} (${own(sorted.at(-1))}) 0.00`;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* One source read, its steps said as they happen, tried once more when it fails. What it came to is told to `onSource` as soon
|
|
259
|
+
* as it is known — `{ id, name, url, ok, steps, ms }` — so a slow source never holds the others back from the screen.
|
|
260
|
+
*/
|
|
261
|
+
async function readSource(source, fetchImpl, onSource, { retryMs = RETRY_MS, maxBytes = MAX_BYTES, sourceMs = SOURCE_MS } = {}) {
|
|
262
|
+
const steps = [];
|
|
263
|
+
const say = (line) => steps.push(line);
|
|
264
|
+
const started = Date.now();
|
|
265
|
+
let got = null;
|
|
266
|
+
for (let attempt = 1; attempt <= 2 && !got; attempt += 1) {
|
|
267
|
+
const tally = {};
|
|
268
|
+
const limits = { maxBytes, sourceMs, deadline: AbortSignal.timeout(sourceMs) };
|
|
269
|
+
try {
|
|
270
|
+
const entries = await READERS[source.id](fetchImpl, say, tally, limits);
|
|
271
|
+
steps.push(...keptLines(entries, tally));
|
|
272
|
+
const none = unplaced(entries);
|
|
273
|
+
got = none ? { error: none } : { entries: placed(entries) };
|
|
274
|
+
say(placeLine(entries));
|
|
275
|
+
} catch (cause) {
|
|
276
|
+
say(`failed: ${failedBecause(cause, limits)}${attempt === 1 ? ` — trying once more in ${retryMs / 1000} s` : ''}`);
|
|
277
|
+
if (attempt === 2) got = { error: failedBecause(cause, limits) };
|
|
278
|
+
else await new Promise((resolve) => setTimeout(resolve, retryMs));
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
onSource?.({ id: source.id, name: source.name, url: source.url, ok: !got.error, steps, ms: Date.now() - started });
|
|
282
|
+
return got;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/** The three sources as they are published now, each read on its own: one that fails is its error, never the others'. */
|
|
286
|
+
async function fetchSources(fetchImpl, onSource, limits) {
|
|
287
|
+
const read = await Promise.all(SOURCES.map((s) => readSource(s, fetchImpl, onSource, limits)));
|
|
288
|
+
return Object.fromEntries(SOURCES.map((s, i) => [s.id, read[i]]));
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/** Under a stand-in (`SUMO_AGENTS_MODEL_CMD`), the sources are what it answers to `{ kind: 'scorecard' }`, entries already read: the suite never reaches the web. */
|
|
292
|
+
function askStandIn(standIn, onSource) {
|
|
293
|
+
const run = spawnSync(standIn, [], { input: JSON.stringify({ kind: 'scorecard' }), encoding: 'utf8', timeout: TIMEOUT_MS });
|
|
294
|
+
let answer = null;
|
|
295
|
+
try {
|
|
296
|
+
answer = JSON.parse(run.stdout);
|
|
297
|
+
} catch {
|
|
298
|
+
// Said below.
|
|
299
|
+
}
|
|
300
|
+
const why = (run.stderr || run.stdout || '').trim().slice(0, 200) || `exit ${run.status}`;
|
|
301
|
+
return Object.fromEntries(
|
|
302
|
+
SOURCES.map((source) => {
|
|
303
|
+
const got = answer && typeof answer === 'object' ? (answer[source.id] ?? { error: 'no answer' }) : { error: why };
|
|
304
|
+
const none = got.error ? null : unplaced(got.entries ?? []);
|
|
305
|
+
const steps = [`asked the stand-in (${standIn})`, ...(got.error ? [`failed: ${got.error}`] : [`kept ${got.entries?.length ?? 0} rows`, placeLine(got.entries ?? [])])];
|
|
306
|
+
onSource?.({ id: source.id, name: source.name, url: source.url, ok: !got.error && !none, steps, ms: 0 });
|
|
307
|
+
return [source.id, got.error ? got : none ? { error: none } : { entries: placed(got.entries) }];
|
|
308
|
+
}),
|
|
309
|
+
);
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
/** The scorecard as saved, or null while none was ever fetched. */
|
|
313
|
+
export function savedScorecard(db) {
|
|
314
|
+
try {
|
|
315
|
+
const card = JSON.parse(getMeta(db, KEY) ?? 'null');
|
|
316
|
+
return card && typeof card === 'object' && card.sources ? card : null;
|
|
317
|
+
} catch {
|
|
318
|
+
return null;
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* Fetches the sources and saves the scorecard: each source's entries for Sumo's models, placed among themselves. A source
|
|
324
|
+
* that cannot be read now keeps what was saved from it last, with the day it was read; one never read is left out. Each source
|
|
325
|
+
* is told to `onSource` as it is read (see `readSource`). Never
|
|
326
|
+
* throws: `ok` is false only when no source has anything for Sumo's models.
|
|
327
|
+
*/
|
|
328
|
+
export async function refreshScorecard(db, { fetchImpl = fetch, now = new Date().toISOString(), onSource = null, retryMs = RETRY_MS, maxBytes = MAX_BYTES, sourceMs = SOURCE_MS } = {}) {
|
|
329
|
+
const standIn = process.env.SUMO_AGENTS_MODEL_CMD;
|
|
330
|
+
const fetched = standIn ? askStandIn(standIn, onSource) : await fetchSources(fetchImpl, onSource, { retryMs, maxBytes, sourceMs });
|
|
331
|
+
const before = savedScorecard(db)?.sources ?? {};
|
|
332
|
+
const sources = {};
|
|
333
|
+
const failed = {};
|
|
334
|
+
for (const { id } of SOURCES) {
|
|
335
|
+
const got = fetched[id] ?? { error: 'no answer' };
|
|
336
|
+
const entries = got.error ? null : got.entries;
|
|
337
|
+
// Each entry keeps the API id of the model it scored: when the catalog moves a name to another version, it is not read for it.
|
|
338
|
+
if (entries && entries.length > 0) sources[id] = { at: now, entries: entries.map((e) => ({ ...e, id: modelId(e.model) })) };
|
|
339
|
+
else {
|
|
340
|
+
failed[id] = got.error ?? 'none of Sumo\'s models is listed';
|
|
341
|
+
if (before[id]) sources[id] = before[id];
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
if (Object.keys(sources).length === 0) return { ok: false, error: Object.values(failed)[0] ?? 'nothing fetched', failed };
|
|
345
|
+
setMeta(db, KEY, JSON.stringify({ at: now, sources }));
|
|
346
|
+
return { ok: true, read: SOURCES.filter((s) => !failed[s.id]).map((s) => s.id), failed, kept: Object.keys(failed).filter((id) => sources[id]) };
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/** A finished refresh in one line, for setup and the settings page. */
|
|
350
|
+
export function scorecardSummary(result) {
|
|
351
|
+
if (!result.ok) return `scorecard not fetched — ${result.error}`;
|
|
352
|
+
const name = (id) => SOURCES.find((s) => s.id === id).name;
|
|
353
|
+
const parts = [`scorecard from ${result.read.map(name).join('; ') || 'no source today'}`];
|
|
354
|
+
const failed = Object.entries(result.failed);
|
|
355
|
+
if (failed.length > 0) parts.push(`not read: ${failed.map(([id, why]) => `${name(id)} (${why}${result.kept.includes(id) ? '; the last copy is kept' : ''})`).join(', ')}`);
|
|
356
|
+
return parts.join(' — ');
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
/**
|
|
360
|
+
* A source's entry for one route: the model at that effort; else, where the source lists the model at other efforts, the
|
|
361
|
+
* nearest of them (the lower on a tie); else the model's own row. Every route of a model so draws on the same sources — a
|
|
362
|
+
* source that lists only some efforts would otherwise lift the routes it leaves out above the ones it ranks. Only entries for
|
|
363
|
+
* the model the catalog runs under that name now count.
|
|
364
|
+
*/
|
|
365
|
+
function entryFor(source, model, effort, efforts) {
|
|
366
|
+
const own = source.entries.filter((e) => e.model === model && e.id === modelId(model));
|
|
367
|
+
const exact = own.find((e) => e.effort === effort);
|
|
368
|
+
if (exact) return exact;
|
|
369
|
+
const at = efforts.indexOf(effort);
|
|
370
|
+
const levels = own.filter((e) => efforts.includes(e.effort)).sort((a, b) => Math.abs(efforts.indexOf(a.effort) - at) - Math.abs(efforts.indexOf(b.effort) - at) || efforts.indexOf(a.effort) - efforts.indexOf(b.effort));
|
|
371
|
+
return levels[0] ?? own.find((e) => e.effort === ANY) ?? null;
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
/**
|
|
375
|
+
* One route on the saved scorecard: its score (the mean of its places, 0–1), the sources behind it, and what one task costs
|
|
376
|
+
* on it where a source says. `efforts` is route.mjs's list, lowest first. Null when no source lists the model.
|
|
377
|
+
*/
|
|
378
|
+
export function routeScore(card, model, effort, efforts) {
|
|
379
|
+
const found = Object.values(card?.sources ?? {}).map((s) => entryFor(s, model, effort, efforts)).filter(Boolean);
|
|
380
|
+
if (found.length === 0) return null;
|
|
381
|
+
// A cost is a cost at this effort: one borrowed from another effort would price the route at the wrong level.
|
|
382
|
+
const cost = found.find((e) => e.cost !== null && (e.effort === effort || e.effort === ANY))?.cost ?? null;
|
|
383
|
+
return { score: found.reduce((sum, e) => sum + e.place, 0) / found.length, sources: found.length, cost };
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
/** One route's cell: its score, and what a task costs on it where a source says. Null when no source lists the model. */
|
|
387
|
+
function cellOf(card, model, effort, efforts) {
|
|
388
|
+
const r = routeScore(card, model, effort, efforts);
|
|
389
|
+
return r && `${r.score.toFixed(2)}${r.cost === null ? '' : ` $${r.cost.toFixed(2)}`}`;
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/** Where the saved scorecard came from: the day it was fetched, and each source, with its own day where it was kept from an earlier one. */
|
|
393
|
+
function fromLine(card) {
|
|
394
|
+
const day = card.at.slice(0, 10);
|
|
395
|
+
const from = SOURCES.filter((s) => card.sources[s.id]).map((s) => (card.sources[s.id].at.slice(0, 10) === day ? s.name : `${s.name} (${card.sources[s.id].at.slice(0, 10)})`));
|
|
396
|
+
return `fetched ${day} from ${from.join('; ')}`;
|
|
397
|
+
}
|
|
398
|
+
const READING = "a score is a model's place among Sumo's models, 0 to 1, averaged over the sources; a cost is one task's";
|
|
399
|
+
|
|
400
|
+
/** What `sumo models` adds: when the scorecard was fetched and from what, then each model's score and cost per task at each effort. */
|
|
401
|
+
export function scorecardLines(db, efforts, models = ALL_MODELS) {
|
|
402
|
+
const card = savedScorecard(db);
|
|
403
|
+
if (!card) return ['scorecard: not fetched — API models › Discover, on the settings page (sumo config), fetches it'];
|
|
404
|
+
const lines = [`scorecard: ${fromLine(card)}`, ` ${READING}`];
|
|
405
|
+
for (const model of models) {
|
|
406
|
+
const cells = efforts.map((effort) => [effort, cellOf(card, model, effort, efforts)]).filter(([, cell]) => cell).map(([effort, cell]) => `${effort} ${cell}`);
|
|
407
|
+
lines.push(` ${model.padEnd(NAME_WIDTH)} ${cells.length > 0 ? cells.join(' · ') : 'not listed by any source'}`);
|
|
408
|
+
}
|
|
409
|
+
return lines;
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
/** What `/scorecard` ends on, in Markdown for the chat to draw: how the refresh went, then the saved scorecard as a table. */
|
|
413
|
+
export function scorecardReport(db, result, efforts, models = ALL_MODELS) {
|
|
414
|
+
const card = savedScorecard(db);
|
|
415
|
+
if (!card) return scorecardSummary(result);
|
|
416
|
+
const rows = models.map((m) => `| ${m} | ${efforts.map((e) => cellOf(card, m, e, efforts) ?? '—').join(' | ')} |`);
|
|
417
|
+
const table = [`| | ${efforts.join(' | ')} |`, `|---|${efforts.map(() => '---').join('|')}|`, ...rows];
|
|
418
|
+
return [scorecardSummary(result), '', `Saved: ${fromLine(card)}.`, '', ...table, '', `${READING[0].toUpperCase()}${READING.slice(1)}.`].join('\n');
|
|
419
|
+
}
|