trodo-cli 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -9
- package/lib/args.js +1 -1
- package/lib/commands/agents.js +8 -1
- package/lib/commands/api.js +37 -0
- package/lib/commands/auth.js +67 -9
- package/lib/commands/conversations.js +49 -0
- package/lib/commands/datasets.js +12 -4
- package/lib/commands/evals.js +122 -1
- package/lib/commands/issues.js +40 -0
- package/lib/commands/prompts.js +125 -0
- package/lib/commands/runs.js +90 -13
- package/lib/commands/setup.js +34 -0
- package/lib/commands/spans.js +38 -0
- package/lib/commands/traces.js +70 -4
- package/lib/commands/users.js +14 -0
- package/lib/common.js +69 -0
- package/lib/config.js +47 -6
- package/lib/http.js +2 -1
- package/lib/main.js +62 -21
- package/lib/output.js +12 -1
- package/package.json +3 -2
- package/skills/trodo/SKILL.md +121 -0
- package/skills/trodo-datasets/SKILL.md +116 -0
package/README.md
CHANGED
|
@@ -1,25 +1,34 @@
|
|
|
1
1
|
# trodo-cli
|
|
2
2
|
|
|
3
|
-
Trodo from the terminal, a script, CI or a coding agent:
|
|
4
|
-
|
|
5
|
-
label datasets, run agents, and export fine-tuning files.
|
|
3
|
+
Trodo from the terminal, a script, CI or a coding agent: read and measure your
|
|
4
|
+
agents' traces, see which eval step failed, triage and close issues, pull and
|
|
5
|
+
push prompts, build and label datasets, run agents, and export fine-tuning files.
|
|
6
6
|
|
|
7
7
|
```bash
|
|
8
8
|
npm i -g trodo-cli
|
|
9
9
|
trodo login # opens the app to connect this terminal
|
|
10
|
-
trodo
|
|
11
|
-
|
|
10
|
+
trodo setup claude # give Claude Code the Trodo skills
|
|
11
|
+
|
|
12
|
+
trodo traces stats --since 7d --compare
|
|
13
|
+
trodo runs list --since 24h --eval "any:failed" --table
|
|
14
|
+
trodo runs tree RUN_ID
|
|
15
|
+
trodo evals verdicts --run RUN_ID --table
|
|
16
|
+
trodo conversations get CONVERSATION_ID --messages
|
|
17
|
+
trodo issues list --status open,regression --table
|
|
18
|
+
trodo prompts pull support-triage --label production -o prompt.json
|
|
19
|
+
trodo prompts push support-triage -f prompt.json --message "why" --label staging
|
|
20
|
+
|
|
12
21
|
trodo traces export --kind spans --since 30d --eval any:passed --format openai --target output -o sft.jsonl
|
|
13
22
|
trodo datasets create "Good refunds" --kind runs --eval "Refund policy:passed" --include-trace
|
|
14
23
|
trodo datasets export "Good refunds" --rows passed --format openai --target output -o train.jsonl
|
|
15
24
|
```
|
|
16
25
|
|
|
17
|
-
`trodo help` lists every command and filter. Output is JSON (
|
|
18
|
-
for
|
|
26
|
+
`trodo help` lists every command and filter. Output is JSON (`--table` for a
|
|
27
|
+
table, `--csv` for CSV; lists take `--all`). Exit codes: 0 ok, 1 API error, 2 usage.
|
|
28
|
+
`trodo api GET /path` reaches any route of the CLI's API.
|
|
19
29
|
|
|
20
30
|
Auth: `trodo login` (browser), `trodo login --key trodo_mk_…`, or
|
|
21
31
|
`TRODO_API_KEY` / `TRODO_API_URL`. Keys live in Settings → MCP; one acts as
|
|
22
32
|
the person who made it, on its team, with its scopes.
|
|
23
33
|
|
|
24
|
-
|
|
25
|
-
the Trodo repo) teaches the whole loop.
|
|
34
|
+
Docs: https://docs.trodo.ai/platform/cli
|
package/lib/args.js
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* `--` ends flags. Everything else is positional.
|
|
7
7
|
*/
|
|
8
8
|
const SHORT = { o: 'out', f: 'file', h: 'help', n: 'limit' };
|
|
9
|
-
const BOOLEAN = new Set(['help', 'json', 'table', 'live', 'include-trace', 'wait', 'yes', 'no-trace', 'no-browser', 'any-span']);
|
|
9
|
+
const BOOLEAN = new Set(['help', 'json', 'table', 'live', 'include-trace', 'wait', 'yes', 'no-trace', 'no-browser', 'any-span', 'tree', 'messages', 'failed', 'passed', 'right', 'wrong', 'remove', 'compare', 'all', 'csv', 'global']);
|
|
10
10
|
|
|
11
11
|
function parse(argv) {
|
|
12
12
|
const flags = {};
|
package/lib/commands/agents.js
CHANGED
|
@@ -16,6 +16,13 @@ async function resolveAgent(api, ref) {
|
|
|
16
16
|
return hit;
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
+
/** trodo agents get AGENT: the agent with its steps, trigger and published version. */
|
|
20
|
+
async function getAgent([ref], flags) {
|
|
21
|
+
const api = client(config.resolve(flags));
|
|
22
|
+
const a = await resolveAgent(api, ref);
|
|
23
|
+
print(await api.get(`/agents/${a.id}`), flags);
|
|
24
|
+
}
|
|
25
|
+
|
|
19
26
|
async function listAgents(_a, flags) {
|
|
20
27
|
const { workflows } = await client(config.resolve(flags)).get('/agents');
|
|
21
28
|
print(workflows || [], flags, [['name', 'Agent', 44], ['status', 'Status', 10], ['id', 'Id', 36], ['updated_at', 'Updated', 24]]);
|
|
@@ -53,4 +60,4 @@ async function execution([id], flags) {
|
|
|
53
60
|
print(await client(config.resolve(flags)).get(`/agents/executions/${encodeURIComponent(id)}`), flags);
|
|
54
61
|
}
|
|
55
62
|
|
|
56
|
-
module.exports = { list: listAgents, run, executions, execution };
|
|
63
|
+
module.exports = { get: getAgent, list: listAgents, run, executions, execution };
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const config = require('../config');
|
|
5
|
+
const { client } = require('../http');
|
|
6
|
+
const { print } = require('../output');
|
|
7
|
+
const { usage } = require('../common');
|
|
8
|
+
const { list: flagList } = require('../args');
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* trodo api METHOD PATH [--param k=v …] [--data JSON | -f body.json]
|
|
12
|
+
* Any /api/v1 route with the signed-in key, for what no command covers yet.
|
|
13
|
+
* Reads are open; the server refuses any write that deletes, archives or replaces:
|
|
14
|
+
* trodo api GET /runs/conversations --param q=alice
|
|
15
|
+
* trodo api POST /datasets --data '{"name":"Scratch"}'
|
|
16
|
+
*/
|
|
17
|
+
async function call([method, path], flags) {
|
|
18
|
+
const m = String(method || '').toUpperCase();
|
|
19
|
+
if (!['GET', 'POST', 'PATCH', 'PUT', 'DELETE'].includes(m) || !path) throw usage('Usage: trodo api GET|POST|PATCH|PUT|DELETE /path [--param k=v] [--data JSON | -f body.json]');
|
|
20
|
+
const query = {};
|
|
21
|
+
for (const kv of flagList(flags.param)) {
|
|
22
|
+
const i = kv.indexOf('=');
|
|
23
|
+
if (i < 1) throw usage(`--param takes k=v, got "${kv}".`);
|
|
24
|
+
query[kv.slice(0, i)] = kv.slice(i + 1);
|
|
25
|
+
}
|
|
26
|
+
let body;
|
|
27
|
+
const raw = flags.data !== undefined && flags.data !== true ? String(flags.data) : (flags.file && flags.file !== true ? fs.readFileSync(String(flags.file), 'utf8') : null);
|
|
28
|
+
if (raw != null) { try { body = JSON.parse(raw); } catch (e) { throw usage(`The body is not JSON: ${e.message}`); } }
|
|
29
|
+
const p = String(path).startsWith('/') ? String(path) : `/${path}`;
|
|
30
|
+
const res = await client(config.resolve(flags)).raw(m, p, { query, body });
|
|
31
|
+
const text = await res.text();
|
|
32
|
+
let data; try { data = text ? JSON.parse(text) : null; } catch { data = text; }
|
|
33
|
+
if (!res.ok) { process.exitCode = 1; process.stderr.write(`trodo: HTTP ${res.status}\n`); }
|
|
34
|
+
if (typeof data === 'string') process.stdout.write(`${data}\n`); else print(data, flags);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
module.exports = { call };
|
package/lib/commands/auth.js
CHANGED
|
@@ -86,22 +86,80 @@ async function login(_args, flags) {
|
|
|
86
86
|
else key = await browserLogin(url, flags);
|
|
87
87
|
if (!key.startsWith('trodo_mk_')) { const e = new Error('That is not a Trodo API key (they start with trodo_mk_).'); e.exitCode = 2; throw e; }
|
|
88
88
|
const me = await client({ url, key }).get('/me');
|
|
89
|
-
|
|
90
|
-
const
|
|
91
|
-
|
|
89
|
+
await adoptOldKey();
|
|
90
|
+
const before = config.teams().filter((t) => String(t.id) !== String(me.team.id));
|
|
91
|
+
const file = config.saveTeam({ id: me.team.id, name: me.team.name || String(me.team.id), key, url });
|
|
92
|
+
process.stderr.write(`Signed in to ${me.team.name} as ${me.user.email}. Saved to ${file}.${before.length ? ` Also signed in to ${before.map((t) => t.name).join(', ')}: \`trodo switch\` moves between them.` : ''}\n`);
|
|
92
93
|
print({ team: me.team, scopes: me.key.scopes }, flags);
|
|
93
94
|
}
|
|
94
95
|
|
|
96
|
+
/**
|
|
97
|
+
* A 0.2 config held one key and no team: ask the server whose it is and put it
|
|
98
|
+
* in the list, so signing in to a second team keeps the first. Silent when the
|
|
99
|
+
* key no longer works (it is then simply dropped by the next login).
|
|
100
|
+
*/
|
|
101
|
+
async function adoptOldKey(keepCurrent = true) {
|
|
102
|
+
const cfg = config.read();
|
|
103
|
+
if (!cfg.api_key || (cfg.teams && Object.keys(cfg.teams).length)) return;
|
|
104
|
+
const url = String(cfg.api_url || config.DEFAULT_URL).replace(/\/+$/, '');
|
|
105
|
+
try {
|
|
106
|
+
const me = await client({ url, key: cfg.api_key }).get('/me');
|
|
107
|
+
const teams = { [String(me.team.id)]: { name: me.team.name || String(me.team.id), api_key: cfg.api_key, api_url: url } };
|
|
108
|
+
config.write({ api_url: url, current_team: keepCurrent ? String(me.team.id) : null, teams });
|
|
109
|
+
} catch { /* an old key that no longer works is not worth keeping */ }
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** trodo teams: the teams this terminal is signed in to; the current one is marked. */
|
|
113
|
+
async function teams(_args, flags) {
|
|
114
|
+
await adoptOldKey();
|
|
115
|
+
const rows = config.teams().map(({ api_key: k, ...t }) => ({ ...t, current: t.current ? '*' : '' }));
|
|
116
|
+
if (!rows.length) {
|
|
117
|
+
const legacy = config.read().api_key;
|
|
118
|
+
process.stderr.write(legacy ? 'Signed in with one key from an older trodo; `trodo login` adds teams by name.\n' : 'Not signed in. `trodo login` adds a team.\n');
|
|
119
|
+
}
|
|
120
|
+
print(flags.table ? rows : rows.map((r) => ({ ...r, current: r.current === '*' })), flags, [['current', '', 1], ['name', 'Team', 40], ['id', 'Id', 8], ['api_url', 'Server', 30]]);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** trodo switch TEAM: the team every command uses from now on (`--team` picks one for a single command). */
|
|
124
|
+
async function switchTeam([ref], flags) {
|
|
125
|
+
if (!ref) { const e = new Error('Usage: trodo switch TEAM (`trodo teams` lists them)'); e.exitCode = 2; throw e; }
|
|
126
|
+
await adoptOldKey();
|
|
127
|
+
const cfg = config.read();
|
|
128
|
+
const t = config.findTeam(cfg, ref);
|
|
129
|
+
if (!t) { const e = new Error(`Not signed in to a team called "${ref}". \`trodo teams\` lists yours; \`trodo login\` adds one.`); e.exitCode = 2; throw e; }
|
|
130
|
+
config.write({ ...cfg, current_team: t.id });
|
|
131
|
+
process.stderr.write(`Now on ${t.name}.\n`);
|
|
132
|
+
print({ team: { id: t.id, name: t.name } }, flags);
|
|
133
|
+
}
|
|
134
|
+
|
|
95
135
|
async function whoami(_args, flags) {
|
|
96
136
|
print(await client(config.resolve(flags)).get('/me'), flags);
|
|
97
137
|
}
|
|
98
138
|
|
|
99
|
-
|
|
139
|
+
/**
|
|
140
|
+
* trodo logout [TEAM] [--all]: forget the current team's key (or TEAM's, or
|
|
141
|
+
* every one). The key itself stays valid until revoked in Settings → MCP.
|
|
142
|
+
*/
|
|
143
|
+
async function logout([ref], flags) {
|
|
100
144
|
const cfg = config.read();
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
145
|
+
const all = config.teams(cfg);
|
|
146
|
+
let removed = [];
|
|
147
|
+
if (flags.all || !all.length) {
|
|
148
|
+
removed = all.map((t) => t.name);
|
|
149
|
+
config.write({ api_url: cfg.api_url });
|
|
150
|
+
} else {
|
|
151
|
+
const t = ref ? config.findTeam(cfg, ref) : all.find((x) => x.current) || all[0];
|
|
152
|
+
if (!t) { const e = new Error(`Not signed in to a team called "${ref}".`); e.exitCode = 2; throw e; }
|
|
153
|
+
const rest = { ...cfg.teams };
|
|
154
|
+
delete rest[t.id];
|
|
155
|
+
const left = Object.keys(rest);
|
|
156
|
+
const current = String(cfg.current_team) === t.id ? (left[0] || null) : cfg.current_team;
|
|
157
|
+
config.write({ api_url: cfg.api_url, current_team: current, teams: rest });
|
|
158
|
+
removed = [t.name];
|
|
159
|
+
if (current && current !== cfg.current_team) process.stderr.write(`Now on ${rest[current].name || current}.\n`);
|
|
160
|
+
}
|
|
161
|
+
process.stderr.write(removed.length ? `Forgot the key for ${removed.join(', ')}.\n` : 'Removed the saved key.\n');
|
|
162
|
+
print({ ok: true, removed }, flags);
|
|
105
163
|
}
|
|
106
164
|
|
|
107
|
-
module.exports = { login, whoami, logout };
|
|
165
|
+
module.exports = { login, whoami, logout, teams, switch: switchTeam };
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const config = require('../config');
|
|
4
|
+
const { client } = require('../http');
|
|
5
|
+
const { print } = require('../output');
|
|
6
|
+
const { buildSpec } = require('./traces');
|
|
7
|
+
const { windowOf } = require('./runs');
|
|
8
|
+
const { usage, listQuery, pages, unitVerdicts } = require('../common');
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* trodo conversations list [filter flags] [--search TEXT] [--limit 50] [--offset N]
|
|
12
|
+
* A conversation (a thread) matches when any of its runs does; eval conditions
|
|
13
|
+
* are decided for the whole conversation. --search matches the id or the user.
|
|
14
|
+
*/
|
|
15
|
+
async function list(_args, flags) {
|
|
16
|
+
const api = client(config.resolve(flags));
|
|
17
|
+
const spec = await buildSpec(api, flags, 'runs');
|
|
18
|
+
const q = listQuery(spec, windowOf);
|
|
19
|
+
if (flags.search && flags.search !== true) q.q = String(flags.search);
|
|
20
|
+
const rows = await pages(flags, async (limit, offset) => { const out = await api.get('/runs/conversations', { ...q, limit, offset }); return out.conversations || []; });
|
|
21
|
+
print(rows, flags, [['conversation_id', 'Conversation', 40], ['distinct_id', 'User', 26], ['run_count', 'Runs', 5], ['error_count', 'Errors', 6], ['last_run_at', 'Last run', 24]]);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* trodo conversations get ID [--messages]: the thread, every run in it in order,
|
|
26
|
+
* and its evaluations. --messages prints the thread as user / assistant turns.
|
|
27
|
+
*/
|
|
28
|
+
async function get([id], flags) {
|
|
29
|
+
if (!id) throw usage('Usage: trodo conversations get CONVERSATION_ID [--messages]');
|
|
30
|
+
const api = client(config.resolve(flags));
|
|
31
|
+
const [detail, verdicts] = await Promise.all([
|
|
32
|
+
api.get(`/runs/conversations/${encodeURIComponent(id)}`),
|
|
33
|
+
api.get(`/eval3/units/conversation/${encodeURIComponent(id)}/verdicts`).catch(() => null),
|
|
34
|
+
]);
|
|
35
|
+
if (flags.messages) {
|
|
36
|
+
const turns = [];
|
|
37
|
+
for (const r of detail.runs || []) {
|
|
38
|
+
const input = r.input ?? r.input_preview ?? null;
|
|
39
|
+
const output = r.output ?? r.output_preview ?? null;
|
|
40
|
+
if (input != null) turns.push({ role: 'user', content: input, run_id: r.run_id, at: r.started_at });
|
|
41
|
+
if (output != null) turns.push({ role: 'assistant', content: output, run_id: r.run_id, at: r.ended_at || r.started_at });
|
|
42
|
+
}
|
|
43
|
+
print({ conversation_id: id, messages: turns }, flags);
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
print({ ...detail, evaluations: unitVerdicts(verdicts) }, flags);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
module.exports = { list, get };
|
package/lib/commands/datasets.js
CHANGED
|
@@ -33,7 +33,7 @@ async function getDataset([ref], flags) {
|
|
|
33
33
|
}
|
|
34
34
|
|
|
35
35
|
/**
|
|
36
|
-
* trodo datasets create NAME [--description D]
|
|
36
|
+
* trodo datasets create NAME [--description D] [-f rows.jsonl]
|
|
37
37
|
* [--kind spans|runs|conversations|verdicts <filter flags> [--live] [--include-trace]]
|
|
38
38
|
* With --kind, the dataset starts with a source: what the filter matches is copied now
|
|
39
39
|
* (and, with --live, every 15 minutes after).
|
|
@@ -43,10 +43,13 @@ async function create([name], flags) {
|
|
|
43
43
|
const api = client(config.resolve(flags));
|
|
44
44
|
// The filter is checked before anything is made: a typo never leaves an empty dataset behind.
|
|
45
45
|
const spec = flags.kind ? await buildSpec(api, flags, String(flags.kind)) : null;
|
|
46
|
+
// So is the file: read and parsed before the dataset exists.
|
|
47
|
+
const rows = flags.file && flags.file !== true ? await readRows(String(flags.file)) : null;
|
|
46
48
|
const { dataset } = await api.post('/datasets', { name, ...(flags.description ? { description: String(flags.description) } : {}) });
|
|
47
49
|
let source = null;
|
|
48
50
|
if (spec) source = await api.post(`/datasets/${dataset.id}/sources`, { ...spec, live: Boolean(flags.live) });
|
|
49
|
-
|
|
51
|
+
const added = rows && rows.length ? await appendRows(api, dataset, rows, flags) : null;
|
|
52
|
+
print({ dataset: { id: dataset.id, name: dataset.name }, ...(source ? { source: source.source, run: source.feed } : {}), ...(added ? { rows: added } : {}) }, flags);
|
|
50
53
|
}
|
|
51
54
|
|
|
52
55
|
/** trodo datasets add DATASET --kind KIND <filter flags> [--live] [--include-trace]: another source. */
|
|
@@ -68,6 +71,11 @@ async function append([ref], flags) {
|
|
|
68
71
|
const api = client(config.resolve(flags));
|
|
69
72
|
const d = await resolveDataset(api, ref);
|
|
70
73
|
const rows = await readRows(flags.file);
|
|
74
|
+
print({ dataset: d.name, ...(await appendRows(api, d, rows, flags)) }, flags);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Rows (plain or references to traffic) into dataset `d`; returns { written, skipped }. */
|
|
78
|
+
async function appendRows(api, d, rows, flags) {
|
|
71
79
|
const REF = new Set(['run_id', 'span_id', 'conversation_id', 'eval_run_id']);
|
|
72
80
|
const refs = { run_ids: [], span_ids: [], conversation_ids: [], eval_run_ids: [] };
|
|
73
81
|
const plain = [];
|
|
@@ -97,7 +105,7 @@ async function append([ref], flags) {
|
|
|
97
105
|
const out = await api.post(`/datasets/${d.id}/items`, { items: part });
|
|
98
106
|
written += Number(out.written ?? out.added ?? (Array.isArray(out.items) ? out.items.length : part.length)); skipped += Number(out.skipped ?? 0);
|
|
99
107
|
}
|
|
100
|
-
|
|
108
|
+
return { written, skipped };
|
|
101
109
|
}
|
|
102
110
|
|
|
103
111
|
/**
|
|
@@ -171,4 +179,4 @@ async function versions([ref], flags) {
|
|
|
171
179
|
print(out.versions || out, flags, [['version_no', 'v', 4], ['name', 'Name', 40], ['item_count', 'Rows', 8], ['created_at', 'At', 24]]);
|
|
172
180
|
}
|
|
173
181
|
|
|
174
|
-
module.exports = { list: listDatasets, get: getDataset, create, add: addSource, append, update, export: exportDataset, rows: exportDataset, versions, resolveDataset, UUID };
|
|
182
|
+
module.exports = { list: listDatasets, get: getDataset, create, add: addSource, append, update, export: exportDataset, rows: exportDataset, download: exportDataset, versions, resolveDataset, UUID };
|
package/lib/commands/evals.js
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
const config = require('../config');
|
|
4
4
|
const { client } = require('../http');
|
|
5
5
|
const { print } = require('../output');
|
|
6
|
+
const { usage, pick, int, unitVerdicts } = require('../common');
|
|
6
7
|
|
|
7
8
|
/** trodo evals list: the names --eval takes, and each eval's steps (STEP.FIELD). */
|
|
8
9
|
async function listEvals(_args, flags) {
|
|
@@ -14,4 +15,124 @@ async function listEvals(_args, flags) {
|
|
|
14
15
|
print(flags.table ? rows : evals, flags, [['name', 'Eval', 44], ['target', 'Target', 12], ['enabled', 'On', 5], ['step_fields', 'Steps (STEP.FIELD)', 70]]);
|
|
15
16
|
}
|
|
16
17
|
|
|
17
|
-
|
|
18
|
+
async function resolveEval(api, ref) {
|
|
19
|
+
const { evals } = await api.get('/evals');
|
|
20
|
+
return pick(evals || [], ref, { kind: 'eval', listCmd: 'trodo evals list' });
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** trodo evals get EVAL: the eval, its production version's graph and its 30-day stats. */
|
|
24
|
+
async function getEval([ref], flags) {
|
|
25
|
+
const api = client(config.resolve(flags));
|
|
26
|
+
const e = await resolveEval(api, ref);
|
|
27
|
+
const [lineage, stats] = await Promise.all([
|
|
28
|
+
api.get(`/eval3/definitions/${e.id}`),
|
|
29
|
+
api.get(`/eval3/definitions/${e.id}/stats`).catch(() => null),
|
|
30
|
+
]);
|
|
31
|
+
print({ ...lineage, steps: e.steps, stats: stats || undefined }, flags);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** "7d" / "48h" / "30" → whole days for the verdict list (1-365). */
|
|
35
|
+
function daysOf(since) {
|
|
36
|
+
if (since == null || since === true) return undefined;
|
|
37
|
+
const m = /^(\d+)\s*([hd]?)$/.exec(String(since).trim());
|
|
38
|
+
if (!m) throw usage('--since takes days or hours here: 7d, 48h, 30.');
|
|
39
|
+
const n = Number(m[1]);
|
|
40
|
+
return Math.min(365, Math.max(1, m[2] === 'h' ? Math.ceil(n / 24) : n));
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* trodo evals verdicts EVAL [--failed | --passed] [--since 7d] [--version N] [--limit 50] [--offset N]
|
|
45
|
+
* trodo evals verdicts --run RUN_ID | --span SPAN_ID | --conversation ID
|
|
46
|
+
* The eval's verdicts, newest first; or every eval's newest verdict on one unit.
|
|
47
|
+
*/
|
|
48
|
+
async function verdicts([ref], flags) {
|
|
49
|
+
const api = client(config.resolve(flags));
|
|
50
|
+
const unit = ['run', 'span', 'conversation'].find((k) => flags[k] && flags[k] !== true);
|
|
51
|
+
const cols = [['id', 'Verdict', 36], ['lineage_name', 'Eval', 34], ['passed', 'Passed', 6], ['score', 'Score', 6], ['reason_code', 'Reason', 22], ['created_at', 'At', 24]];
|
|
52
|
+
if (unit) {
|
|
53
|
+
const out = await api.get(`/eval3/units/${unit}/${encodeURIComponent(String(flags[unit]))}/verdicts`);
|
|
54
|
+
print(unitVerdicts(out), flags, cols);
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
if (flags.failed && flags.passed) throw usage('Give --failed or --passed, not both.');
|
|
58
|
+
const e = await resolveEval(api, ref);
|
|
59
|
+
const q = {
|
|
60
|
+
limit: int(flags.limit, 50, 200),
|
|
61
|
+
offset: flags.offset,
|
|
62
|
+
days: daysOf(flags.since),
|
|
63
|
+
version: flags.version,
|
|
64
|
+
...(flags.failed ? { passed: 'false' } : flags.passed ? { passed: 'true' } : {}),
|
|
65
|
+
};
|
|
66
|
+
const out = await api.get(`/eval3/definitions/${e.id}/runs`, q);
|
|
67
|
+
const rows = (out.runs || out.verdicts || []).map((r) => ({ lineage_name: e.name, ...r }));
|
|
68
|
+
print(flags.table ? rows : { eval: e.name, ...out }, flags, cols);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** trodo evals verdict VERDICT_ID: one verdict, every step it took and why. */
|
|
72
|
+
async function verdict([id], flags) {
|
|
73
|
+
if (!id) throw usage('Usage: trodo evals verdict VERDICT_ID');
|
|
74
|
+
print(await client(config.resolve(flags)).get(`/eval3/runs/${encodeURIComponent(id)}`), flags);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* trodo evals mark VERDICT_ID --right
|
|
79
|
+
* trodo evals mark VERDICT_ID --wrong --should pass|fail --reason TEXT
|
|
80
|
+
* A person's thumbs on a verdict. A thumbs-down says what it should have been and
|
|
81
|
+
* why; it becomes that verdict row's Expected and feeds the eval's improvement.
|
|
82
|
+
*/
|
|
83
|
+
async function mark([id], flags) {
|
|
84
|
+
if (!id) throw usage('Usage: trodo evals mark VERDICT_ID --right | --wrong --should pass|fail --reason TEXT');
|
|
85
|
+
if (Boolean(flags.right) === Boolean(flags.wrong)) throw usage('Give --right or --wrong.');
|
|
86
|
+
const body = { source: 'v3', eval_run_id: String(id), thumbs: flags.right ? 'up' : 'down' };
|
|
87
|
+
if (flags.wrong) {
|
|
88
|
+
const should = String(flags.should || '');
|
|
89
|
+
if (!['pass', 'fail'].includes(should)) throw usage('--wrong needs --should pass or --should fail.');
|
|
90
|
+
if (!flags.reason || flags.reason === true) throw usage('--wrong needs --reason "why".');
|
|
91
|
+
body.corrected_passed = should === 'pass';
|
|
92
|
+
body.reason_text = String(flags.reason);
|
|
93
|
+
} else if (flags.reason && flags.reason !== true) body.reason_text = String(flags.reason);
|
|
94
|
+
print(await client(config.resolve(flags)).post('/eval3/feedback', body), flags);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* trodo evals queue [--eval EVAL] [--graded] [--limit 50]
|
|
99
|
+
* Verdicts waiting on a person: an eval's human step stops there until someone
|
|
100
|
+
* picks one of its choices.
|
|
101
|
+
*/
|
|
102
|
+
async function queue(_a, flags) {
|
|
103
|
+
const api = client(config.resolve(flags));
|
|
104
|
+
const lineageId = flags.eval && flags.eval !== true ? (await resolveEval(api, flags.eval)).id : undefined;
|
|
105
|
+
const out = await api.get('/eval3/queue', { status: flags.graded ? 'graded' : 'open', lineage_id: lineageId, limit: int(flags.limit, 50, 200), offset: flags.offset });
|
|
106
|
+
print(out.items || out, flags, [['id', 'Item', 36], ['lineage_name', 'Eval', 34], ['node_id', 'Step', 22], ['status', 'Status', 8], ['created_at', 'Waiting since', 24]]);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** trodo evals review ITEM_ID: one waiting item with the path the eval took to it. */
|
|
110
|
+
async function review([id], flags) {
|
|
111
|
+
if (!id) throw usage('Usage: trodo evals review ITEM_ID');
|
|
112
|
+
print(await client(config.resolve(flags)).get(`/eval3/queue/${encodeURIComponent(id)}`), flags);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** trodo evals grade ITEM_ID CHOICE: answer the human step; the eval carries on from there. */
|
|
116
|
+
async function grade([id, choice], flags) {
|
|
117
|
+
if (!id || !choice) throw usage('Usage: trodo evals grade ITEM_ID CHOICE (`trodo evals review ITEM_ID` lists the choices)');
|
|
118
|
+
print(await client(config.resolve(flags)).post(`/eval3/queue/${encodeURIComponent(id)}/grade`, { choice: String(choice) }), flags);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* trodo evals steps EVAL [--since 30d] [--version N]
|
|
123
|
+
* The paths verdicts took through the eval's steps (its nodes), each with how
|
|
124
|
+
* many runs took it and how they came out: which step decides most failures.
|
|
125
|
+
*/
|
|
126
|
+
async function steps([ref], flags) {
|
|
127
|
+
const api = client(config.resolve(flags));
|
|
128
|
+
const e = await resolveEval(api, ref);
|
|
129
|
+
const out = await api.get(`/eval3/definitions/${e.id}/steps`, { days: daysOf(flags.since) || 30, version: flags.version });
|
|
130
|
+
if (flags.table) {
|
|
131
|
+
const rows = (out.paths || []).map((p) => ({ path: (p.names || p.path || []).join(' → '), runs: p.runs, share: p.share, passed: p.passed, failed: p.failed, errored: p.errored }));
|
|
132
|
+
print(rows, flags, [['path', 'Path through the steps', 90], ['runs', 'Runs', 6], ['share', 'Share', 6], ['passed', 'Passed', 6], ['failed', 'Failed', 6], ['errored', 'Errored', 7]]);
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
print({ eval: e.name, steps: e.steps, ...out }, flags);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
module.exports = { list: listEvals, get: getEval, verdicts, verdict, mark, queue, review, grade, steps, daysOf };
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const config = require('../config');
|
|
4
|
+
const { client } = require('../http');
|
|
5
|
+
const { print } = require('../output');
|
|
6
|
+
const { usage, int } = require('../common');
|
|
7
|
+
|
|
8
|
+
const csv = (v) => (v === undefined || v === true ? undefined : [].concat(v).join(','));
|
|
9
|
+
|
|
10
|
+
/** trodo issues list [--status open,regression,closed] [--agent NAME] [--search TEXT] [--limit 50] */
|
|
11
|
+
async function list(_a, flags) {
|
|
12
|
+
const out = await client(config.resolve(flags)).get('/issues', {
|
|
13
|
+
status: csv(flags.status), agent: csv(flags.agent), search: flags.search !== true ? flags.search : undefined,
|
|
14
|
+
limit: int(flags.limit, 50, 200), offset: flags.offset,
|
|
15
|
+
});
|
|
16
|
+
print(out.issues || out, flags, [['id', 'Id', 6], ['status', 'Status', 10], ['agent_name', 'Agent', 28], ['title', 'Issue', 60], ['evidence_count', 'Evidence', 8], ['last_seen_at', 'Last seen', 24]]);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** trodo issues get ID: the issue, its evidence (runs, spans, conversations) and its timeline. */
|
|
20
|
+
async function get([id], flags) {
|
|
21
|
+
if (!id) throw usage('Usage: trodo issues get ISSUE_ID');
|
|
22
|
+
print(await client(config.resolve(flags)).get(`/issues/${encodeURIComponent(id)}`), flags);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** trodo issues failures ID: the recorded inputs of every failure, to replay against a fix. */
|
|
26
|
+
async function failures([id], flags) {
|
|
27
|
+
if (!id) throw usage('Usage: trodo issues failures ISSUE_ID');
|
|
28
|
+
print(await client(config.resolve(flags)).get(`/issues/${encodeURIComponent(id)}/failures`, { limit: flags.limit, cursor: flags.cursor !== true ? flags.cursor : undefined }), flags);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
async function setStatus(id, status, flags) {
|
|
32
|
+
if (!id) throw usage(`Usage: trodo issues ${status === 'closed' ? 'close' : 'reopen'} ISSUE_ID`);
|
|
33
|
+
print(await client(config.resolve(flags)).post(`/issues/${encodeURIComponent(id)}/status`, { status }), flags);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** trodo issues close ID: closed; a new failure of the same kind reopens it as a regression. */
|
|
37
|
+
const close = ([id], flags) => setStatus(id, 'closed', flags);
|
|
38
|
+
const reopen = ([id], flags) => setStatus(id, 'open', flags);
|
|
39
|
+
|
|
40
|
+
module.exports = { list, get, failures, close, reopen };
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const config = require('../config');
|
|
5
|
+
const { client } = require('../http');
|
|
6
|
+
const { print } = require('../output');
|
|
7
|
+
const { usage, pick } = require('../common');
|
|
8
|
+
|
|
9
|
+
/** What a prompt version is, without who saved it: what `pull` writes and `push` reads. */
|
|
10
|
+
const CONTENT_KEYS = ['messages', 'model', 'tools', 'response_format', 'variables'];
|
|
11
|
+
|
|
12
|
+
/** A refusal to edit, said as what to do about it. */
|
|
13
|
+
async function editing(prompt, fn) {
|
|
14
|
+
try { return await fn(); } catch (e) {
|
|
15
|
+
if (e && e.status === 403 && e.body && e.body.error === 'forbidden_edit') {
|
|
16
|
+
throw Object.assign(new Error(`You can't change "${prompt}": only its owner and the editors they add can. Ask the owner to add you (prompt page → Editors).`), { exitCode: 1 });
|
|
17
|
+
}
|
|
18
|
+
throw e;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
async function resolvePrompt(api, ref) {
|
|
23
|
+
const { prompts } = await api.get('/prompts');
|
|
24
|
+
const hit = pick(prompts || [], ref, { kind: 'prompt', listCmd: 'trodo prompts list' });
|
|
25
|
+
const { prompt } = await api.get(`/prompts/${hit.id}`);
|
|
26
|
+
return prompt;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** The version a --version N or --label L names; the newest one otherwise. */
|
|
30
|
+
function versionOf(prompt, flags) {
|
|
31
|
+
const versions = prompt.versions || [];
|
|
32
|
+
if (flags.version !== undefined && flags.version !== true) {
|
|
33
|
+
const v = versions.find((x) => String(x.version_no) === String(flags.version));
|
|
34
|
+
if (!v) throw usage(`"${prompt.name}" has no version ${flags.version}. \`trodo prompts versions "${prompt.name}"\` lists them.`);
|
|
35
|
+
return v;
|
|
36
|
+
}
|
|
37
|
+
if (flags.label !== undefined && flags.label !== true) {
|
|
38
|
+
const v = versions.find((x) => (x.labels || []).includes(String(flags.label)));
|
|
39
|
+
if (!v) throw usage(`No version of "${prompt.name}" has the label "${flags.label}".`);
|
|
40
|
+
return v;
|
|
41
|
+
}
|
|
42
|
+
return versions.reduce((a, b) => (!a || Number(b.version_no) > Number(a.version_no) ? b : a), null);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
async function list(_a, flags) {
|
|
46
|
+
const { prompts } = await client(config.resolve(flags)).get('/prompts', { label: flags.label !== true ? flags.label : undefined, tag: flags.tag !== true ? flags.tag : undefined });
|
|
47
|
+
print(prompts || [], flags, [['name', 'Prompt', 36], ['current_version', 'Version', 8], ['labels', 'Labels', 24], ['id', 'Id', 36], ['updated_at', 'Updated', 24]]);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** trodo prompts get PROMPT [--version N | --label production]: one version, whole. */
|
|
51
|
+
async function get([ref], flags) {
|
|
52
|
+
const prompt = await resolvePrompt(client(config.resolve(flags)), ref);
|
|
53
|
+
const v = versionOf(prompt, flags);
|
|
54
|
+
print({ id: prompt.id, name: prompt.name, description: prompt.description, tags: prompt.tags, version: v }, flags);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async function versions([ref], flags) {
|
|
58
|
+
const prompt = await resolvePrompt(client(config.resolve(flags)), ref);
|
|
59
|
+
const rows = (prompt.versions || []).map((v) => ({ version_no: v.version_no, labels: v.labels, commit_message: v.commit_message, created_by: v.created_by_email, created_at: v.created_at, content_hash: v.content_hash }));
|
|
60
|
+
print(rows, flags, [['version_no', 'v', 4], ['labels', 'Labels', 22], ['commit_message', 'Message', 44], ['created_by', 'By', 24], ['created_at', 'At', 24]]);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** trodo prompts pull PROMPT [--version N | --label L] [-o prompt.json]: the content to edit and push back. */
|
|
64
|
+
async function pull([ref], flags) {
|
|
65
|
+
const prompt = await resolvePrompt(client(config.resolve(flags)), ref);
|
|
66
|
+
const v = versionOf(prompt, flags);
|
|
67
|
+
const out = { name: prompt.name, version_no: v.version_no };
|
|
68
|
+
for (const k of CONTENT_KEYS) out[k] = v[k];
|
|
69
|
+
if (flags.out && flags.out !== true) {
|
|
70
|
+
fs.writeFileSync(String(flags.out), `${JSON.stringify(out, null, 2)}\n`);
|
|
71
|
+
process.stderr.write(`${prompt.name} v${v.version_no} → ${flags.out}\n`);
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
print(out, flags);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* trodo prompts push PROMPT -f prompt.json [--message "why"] [--label staging]
|
|
79
|
+
* A new version from a file `pull` wrote (or any { messages, model?, tools?,
|
|
80
|
+
* variables? }). The prompt is made when no prompt has that name. Content equal
|
|
81
|
+
* to the newest version cuts no version.
|
|
82
|
+
*/
|
|
83
|
+
async function push([ref], flags) {
|
|
84
|
+
const file = flags.file;
|
|
85
|
+
if (!file || file === true) throw usage('Usage: trodo prompts push PROMPT -f prompt.json [--message "why"] [--label staging]');
|
|
86
|
+
let content;
|
|
87
|
+
try { content = JSON.parse(fs.readFileSync(String(file), 'utf8')); } catch (e) { throw usage(`${file} is not a JSON prompt: ${e.message}`); }
|
|
88
|
+
if (!Array.isArray(content.messages) || !content.messages.length) throw usage(`${file} has no messages.`);
|
|
89
|
+
const api = client(config.resolve(flags));
|
|
90
|
+
const name = ref && ref !== true ? String(ref) : content.name;
|
|
91
|
+
if (!name) throw usage('Name the prompt.');
|
|
92
|
+
const { prompts } = await api.get('/prompts');
|
|
93
|
+
let target = (prompts || []).find((p) => p.id === name || p.name === name);
|
|
94
|
+
let created = false;
|
|
95
|
+
if (!target) {
|
|
96
|
+
const made = await api.post('/prompts', { name, ...(content.description ? { description: content.description } : {}) });
|
|
97
|
+
target = made.prompt || made;
|
|
98
|
+
created = true;
|
|
99
|
+
}
|
|
100
|
+
const body = { messages: content.messages };
|
|
101
|
+
for (const k of ['model', 'tools', 'response_format', 'variables']) if (content[k] !== undefined) body[k] = content[k];
|
|
102
|
+
if (flags.message && flags.message !== true) body.commit_message = String(flags.message);
|
|
103
|
+
if (flags.label && flags.label !== true) body.labels = [].concat(flags.label).map(String);
|
|
104
|
+
const out = await editing(name, () => api.post(`/prompts/${target.id}/versions`, body));
|
|
105
|
+
const v = out.version || {};
|
|
106
|
+
print({ prompt: name, id: target.id, created, version_no: v.version_no, labels: v.labels, unchanged: Boolean(out.unchanged) }, flags);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* trodo prompts label PROMPT VERSION LABEL [--remove]
|
|
111
|
+
* Moves LABEL (say production) to VERSION. Agents that fetch the prompt by that
|
|
112
|
+
* label run the new version from their next call.
|
|
113
|
+
*/
|
|
114
|
+
async function label([ref, versionNo, name], flags) {
|
|
115
|
+
if (!ref || !versionNo || !name) throw usage('Usage: trodo prompts label PROMPT VERSION LABEL [--remove]');
|
|
116
|
+
const api = client(config.resolve(flags));
|
|
117
|
+
const prompt = await resolvePrompt(api, ref);
|
|
118
|
+
const v = versionOf(prompt, { version: versionNo });
|
|
119
|
+
const out = await editing(prompt.name, () => (flags.remove
|
|
120
|
+
? api.del(`/prompts/${prompt.id}/versions/${v.version_no}/label/${encodeURIComponent(String(name))}`)
|
|
121
|
+
: api.post(`/prompts/${prompt.id}/versions/${v.version_no}/label`, { label: String(name) })));
|
|
122
|
+
print({ prompt: prompt.name, version_no: v.version_no, ...(out.version ? { labels: out.version.labels } : out) }, flags);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
module.exports = { list, get, versions, pull, push, label };
|