trodo-cli 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,25 +1,34 @@
1
1
  # trodo-cli
2
2
 
3
- Trodo from the terminal, a script, CI or a coding agent: pull traces by what
4
- your evals said about them, keep whole runs with their span trees, build and
5
- label datasets, run agents, and export fine-tuning files.
3
+ Trodo from the terminal, a script, CI or a coding agent: read and measure your
4
+ agents' traces, see which eval step failed, triage and close issues, pull and
5
+ push prompts, build and label datasets, run agents, and export fine-tuning files.
6
6
 
7
7
  ```bash
8
8
  npm i -g trodo-cli
9
9
  trodo login # opens the app to connect this terminal
10
- trodo evals list --table
11
- trodo traces count --kind runs --since 30d --eval "any:passed" --eval "any:not-failed"
10
+ trodo setup claude # give Claude Code the Trodo skills
11
+
12
+ trodo traces stats --since 7d --compare
13
+ trodo runs list --since 24h --eval "any:failed" --table
14
+ trodo runs tree RUN_ID
15
+ trodo evals verdicts --run RUN_ID --table
16
+ trodo conversations get CONVERSATION_ID --messages
17
+ trodo issues list --status open,regression --table
18
+ trodo prompts pull support-triage --label production -o prompt.json
19
+ trodo prompts push support-triage -f prompt.json --message "why" --label staging
20
+
12
21
  trodo traces export --kind spans --since 30d --eval any:passed --format openai --target output -o sft.jsonl
13
22
  trodo datasets create "Good refunds" --kind runs --eval "Refund policy:passed" --include-trace
14
23
  trodo datasets export "Good refunds" --rows passed --format openai --target output -o train.jsonl
15
24
  ```
16
25
 
17
- `trodo help` lists every command and filter. Output is JSON (add `--table`
18
- for a table). Exit codes: 0 ok, 1 API error, 2 usage.
26
+ `trodo help` lists every command and filter. Output is JSON (`--table` for a
27
+ table, `--csv` for CSV; lists take `--all`). Exit codes: 0 ok, 1 API error, 2 usage.
28
+ `trodo api GET /path` reaches any route of the CLI's API.
19
29
 
20
30
  Auth: `trodo login` (browser), `trodo login --key trodo_mk_…`, or
21
31
  `TRODO_API_KEY` / `TRODO_API_URL`. Keys live in Settings → MCP; one acts as
22
32
  the person who made it, on its team, with its scopes.
23
33
 
24
- Coding agents: the `trodo-datasets` skill (skills/trodo-datasets/SKILL.md in
25
- the Trodo repo) teaches the whole loop.
34
+ Docs: https://docs.trodo.ai/platform/cli
package/lib/args.js CHANGED
@@ -6,7 +6,7 @@
6
6
  * `--` ends flags. Everything else is positional.
7
7
  */
8
8
  const SHORT = { o: 'out', f: 'file', h: 'help', n: 'limit' };
9
- const BOOLEAN = new Set(['help', 'json', 'table', 'live', 'include-trace', 'wait', 'yes', 'no-trace', 'no-browser', 'any-span']);
9
+ const BOOLEAN = new Set(['help', 'json', 'table', 'live', 'include-trace', 'wait', 'yes', 'no-trace', 'no-browser', 'any-span', 'tree', 'messages', 'failed', 'passed', 'right', 'wrong', 'remove', 'compare', 'all', 'csv', 'global']);
10
10
 
11
11
  function parse(argv) {
12
12
  const flags = {};
@@ -16,6 +16,13 @@ async function resolveAgent(api, ref) {
16
16
  return hit;
17
17
  }
18
18
 
19
+ /** trodo agents get AGENT: the agent with its steps, trigger and published version. */
20
+ async function getAgent([ref], flags) {
21
+ const api = client(config.resolve(flags));
22
+ const a = await resolveAgent(api, ref);
23
+ print(await api.get(`/agents/${a.id}`), flags);
24
+ }
25
+
19
26
  async function listAgents(_a, flags) {
20
27
  const { workflows } = await client(config.resolve(flags)).get('/agents');
21
28
  print(workflows || [], flags, [['name', 'Agent', 44], ['status', 'Status', 10], ['id', 'Id', 36], ['updated_at', 'Updated', 24]]);
@@ -53,4 +60,4 @@ async function execution([id], flags) {
53
60
  print(await client(config.resolve(flags)).get(`/agents/executions/${encodeURIComponent(id)}`), flags);
54
61
  }
55
62
 
56
- module.exports = { list: listAgents, run, executions, execution };
63
+ module.exports = { get: getAgent, list: listAgents, run, executions, execution };
@@ -0,0 +1,37 @@
1
+ 'use strict';
2
+
3
+ const fs = require('fs');
4
+ const config = require('../config');
5
+ const { client } = require('../http');
6
+ const { print } = require('../output');
7
+ const { usage } = require('../common');
8
+ const { list: flagList } = require('../args');
9
+
10
+ /**
11
+ * trodo api METHOD PATH [--param k=v …] [--data JSON | -f body.json]
12
+ * Any /api/v1 route with the signed-in key, for what no command covers yet.
13
+ * Reads are open; the server refuses any write that deletes, archives or replaces:
14
+ * trodo api GET /runs/conversations --param q=alice
15
+ * trodo api POST /datasets --data '{"name":"Scratch"}'
16
+ */
17
+ async function call([method, path], flags) {
18
+ const m = String(method || '').toUpperCase();
19
+ if (!['GET', 'POST', 'PATCH', 'PUT', 'DELETE'].includes(m) || !path) throw usage('Usage: trodo api GET|POST|PATCH|PUT|DELETE /path [--param k=v] [--data JSON | -f body.json]');
20
+ const query = {};
21
+ for (const kv of flagList(flags.param)) {
22
+ const i = kv.indexOf('=');
23
+ if (i < 1) throw usage(`--param takes k=v, got "${kv}".`);
24
+ query[kv.slice(0, i)] = kv.slice(i + 1);
25
+ }
26
+ let body;
27
+ const raw = flags.data !== undefined && flags.data !== true ? String(flags.data) : (flags.file && flags.file !== true ? fs.readFileSync(String(flags.file), 'utf8') : null);
28
+ if (raw != null) { try { body = JSON.parse(raw); } catch (e) { throw usage(`The body is not JSON: ${e.message}`); } }
29
+ const p = String(path).startsWith('/') ? String(path) : `/${path}`;
30
+ const res = await client(config.resolve(flags)).raw(m, p, { query, body });
31
+ const text = await res.text();
32
+ let data; try { data = text ? JSON.parse(text) : null; } catch { data = text; }
33
+ if (!res.ok) { process.exitCode = 1; process.stderr.write(`trodo: HTTP ${res.status}\n`); }
34
+ if (typeof data === 'string') process.stdout.write(`${data}\n`); else print(data, flags);
35
+ }
36
+
37
+ module.exports = { call };
@@ -86,22 +86,80 @@ async function login(_args, flags) {
86
86
  else key = await browserLogin(url, flags);
87
87
  if (!key.startsWith('trodo_mk_')) { const e = new Error('That is not a Trodo API key (they start with trodo_mk_).'); e.exitCode = 2; throw e; }
88
88
  const me = await client({ url, key }).get('/me');
89
- const cfg = config.read();
90
- const file = config.write({ ...cfg, api_key: key, api_url: url });
91
- process.stderr.write(`Signed in to ${me.team.name} as ${me.user.email}. Saved to ${file}.\n`);
89
+ await adoptOldKey();
90
+ const before = config.teams().filter((t) => String(t.id) !== String(me.team.id));
91
+ const file = config.saveTeam({ id: me.team.id, name: me.team.name || String(me.team.id), key, url });
92
+ process.stderr.write(`Signed in to ${me.team.name} as ${me.user.email}. Saved to ${file}.${before.length ? ` Also signed in to ${before.map((t) => t.name).join(', ')}: \`trodo switch\` moves between them.` : ''}\n`);
92
93
  print({ team: me.team, scopes: me.key.scopes }, flags);
93
94
  }
94
95
 
96
+ /**
97
+ * A 0.2 config held one key and no team: ask the server whose it is and put it
98
+ * in the list, so signing in to a second team keeps the first. Silent when the
99
+ * key no longer works (it is then simply dropped by the next login).
100
+ */
101
+ async function adoptOldKey(keepCurrent = true) {
102
+ const cfg = config.read();
103
+ if (!cfg.api_key || (cfg.teams && Object.keys(cfg.teams).length)) return;
104
+ const url = String(cfg.api_url || config.DEFAULT_URL).replace(/\/+$/, '');
105
+ try {
106
+ const me = await client({ url, key: cfg.api_key }).get('/me');
107
+ const teams = { [String(me.team.id)]: { name: me.team.name || String(me.team.id), api_key: cfg.api_key, api_url: url } };
108
+ config.write({ api_url: url, current_team: keepCurrent ? String(me.team.id) : null, teams });
109
+ } catch { /* an old key that no longer works is not worth keeping */ }
110
+ }
111
+
112
+ /** trodo teams: the teams this terminal is signed in to; the current one is marked. */
113
+ async function teams(_args, flags) {
114
+ await adoptOldKey();
115
+ const rows = config.teams().map(({ api_key: k, ...t }) => ({ ...t, current: t.current ? '*' : '' }));
116
+ if (!rows.length) {
117
+ const legacy = config.read().api_key;
118
+ process.stderr.write(legacy ? 'Signed in with one key from an older trodo; `trodo login` adds teams by name.\n' : 'Not signed in. `trodo login` adds a team.\n');
119
+ }
120
+ print(flags.table ? rows : rows.map((r) => ({ ...r, current: r.current === '*' })), flags, [['current', '', 1], ['name', 'Team', 40], ['id', 'Id', 8], ['api_url', 'Server', 30]]);
121
+ }
122
+
123
+ /** trodo switch TEAM: the team every command uses from now on (`--team` picks one for a single command). */
124
+ async function switchTeam([ref], flags) {
125
+ if (!ref) { const e = new Error('Usage: trodo switch TEAM (`trodo teams` lists them)'); e.exitCode = 2; throw e; }
126
+ await adoptOldKey();
127
+ const cfg = config.read();
128
+ const t = config.findTeam(cfg, ref);
129
+ if (!t) { const e = new Error(`Not signed in to a team called "${ref}". \`trodo teams\` lists yours; \`trodo login\` adds one.`); e.exitCode = 2; throw e; }
130
+ config.write({ ...cfg, current_team: t.id });
131
+ process.stderr.write(`Now on ${t.name}.\n`);
132
+ print({ team: { id: t.id, name: t.name } }, flags);
133
+ }
134
+
95
135
  async function whoami(_args, flags) {
96
136
  print(await client(config.resolve(flags)).get('/me'), flags);
97
137
  }
98
138
 
99
- async function logout(_args, flags) {
139
+ /**
140
+ * trodo logout [TEAM] [--all]: forget the current team's key (or TEAM's, or
141
+ * every one). The key itself stays valid until revoked in Settings → MCP.
142
+ */
143
+ async function logout([ref], flags) {
100
144
  const cfg = config.read();
101
- delete cfg.api_key;
102
- config.write(cfg);
103
- process.stderr.write('Removed the saved key.\n');
104
- print({ ok: true }, flags);
145
+ const all = config.teams(cfg);
146
+ let removed = [];
147
+ if (flags.all || !all.length) {
148
+ removed = all.map((t) => t.name);
149
+ config.write({ api_url: cfg.api_url });
150
+ } else {
151
+ const t = ref ? config.findTeam(cfg, ref) : all.find((x) => x.current) || all[0];
152
+ if (!t) { const e = new Error(`Not signed in to a team called "${ref}".`); e.exitCode = 2; throw e; }
153
+ const rest = { ...cfg.teams };
154
+ delete rest[t.id];
155
+ const left = Object.keys(rest);
156
+ const current = String(cfg.current_team) === t.id ? (left[0] || null) : cfg.current_team;
157
+ config.write({ api_url: cfg.api_url, current_team: current, teams: rest });
158
+ removed = [t.name];
159
+ if (current && current !== cfg.current_team) process.stderr.write(`Now on ${rest[current].name || current}.\n`);
160
+ }
161
+ process.stderr.write(removed.length ? `Forgot the key for ${removed.join(', ')}.\n` : 'Removed the saved key.\n');
162
+ print({ ok: true, removed }, flags);
105
163
  }
106
164
 
107
- module.exports = { login, whoami, logout };
165
+ module.exports = { login, whoami, logout, teams, switch: switchTeam };
@@ -0,0 +1,49 @@
1
+ 'use strict';
2
+
3
+ const config = require('../config');
4
+ const { client } = require('../http');
5
+ const { print } = require('../output');
6
+ const { buildSpec } = require('./traces');
7
+ const { windowOf } = require('./runs');
8
+ const { usage, listQuery, pages, unitVerdicts } = require('../common');
9
+
10
+ /**
11
+ * trodo conversations list [filter flags] [--search TEXT] [--limit 50] [--offset N]
12
+ * A conversation (a thread) matches when any of its runs does; eval conditions
13
+ * are decided for the whole conversation. --search matches the id or the user.
14
+ */
15
+ async function list(_args, flags) {
16
+ const api = client(config.resolve(flags));
17
+ const spec = await buildSpec(api, flags, 'runs');
18
+ const q = listQuery(spec, windowOf);
19
+ if (flags.search && flags.search !== true) q.q = String(flags.search);
20
+ const rows = await pages(flags, async (limit, offset) => { const out = await api.get('/runs/conversations', { ...q, limit, offset }); return out.conversations || []; });
21
+ print(rows, flags, [['conversation_id', 'Conversation', 40], ['distinct_id', 'User', 26], ['run_count', 'Runs', 5], ['error_count', 'Errors', 6], ['last_run_at', 'Last run', 24]]);
22
+ }
23
+
24
+ /**
25
+ * trodo conversations get ID [--messages]: the thread, every run in it in order,
26
+ * and its evaluations. --messages prints the thread as user / assistant turns.
27
+ */
28
+ async function get([id], flags) {
29
+ if (!id) throw usage('Usage: trodo conversations get CONVERSATION_ID [--messages]');
30
+ const api = client(config.resolve(flags));
31
+ const [detail, verdicts] = await Promise.all([
32
+ api.get(`/runs/conversations/${encodeURIComponent(id)}`),
33
+ api.get(`/eval3/units/conversation/${encodeURIComponent(id)}/verdicts`).catch(() => null),
34
+ ]);
35
+ if (flags.messages) {
36
+ const turns = [];
37
+ for (const r of detail.runs || []) {
38
+ const input = r.input ?? r.input_preview ?? null;
39
+ const output = r.output ?? r.output_preview ?? null;
40
+ if (input != null) turns.push({ role: 'user', content: input, run_id: r.run_id, at: r.started_at });
41
+ if (output != null) turns.push({ role: 'assistant', content: output, run_id: r.run_id, at: r.ended_at || r.started_at });
42
+ }
43
+ print({ conversation_id: id, messages: turns }, flags);
44
+ return;
45
+ }
46
+ print({ ...detail, evaluations: unitVerdicts(verdicts) }, flags);
47
+ }
48
+
49
+ module.exports = { list, get };
@@ -33,7 +33,7 @@ async function getDataset([ref], flags) {
33
33
  }
34
34
 
35
35
  /**
36
- * trodo datasets create NAME [--description D]
36
+ * trodo datasets create NAME [--description D] [-f rows.jsonl]
37
37
  * [--kind spans|runs|conversations|verdicts <filter flags> [--live] [--include-trace]]
38
38
  * With --kind, the dataset starts with a source: what the filter matches is copied now
39
39
  * (and, with --live, every 15 minutes after).
@@ -43,10 +43,13 @@ async function create([name], flags) {
43
43
  const api = client(config.resolve(flags));
44
44
  // The filter is checked before anything is made: a typo never leaves an empty dataset behind.
45
45
  const spec = flags.kind ? await buildSpec(api, flags, String(flags.kind)) : null;
46
+ // So is the file: read and parsed before the dataset exists.
47
+ const rows = flags.file && flags.file !== true ? await readRows(String(flags.file)) : null;
46
48
  const { dataset } = await api.post('/datasets', { name, ...(flags.description ? { description: String(flags.description) } : {}) });
47
49
  let source = null;
48
50
  if (spec) source = await api.post(`/datasets/${dataset.id}/sources`, { ...spec, live: Boolean(flags.live) });
49
- print({ dataset: { id: dataset.id, name: dataset.name }, ...(source ? { source: source.source, run: source.feed } : {}) }, flags);
51
+ const added = rows && rows.length ? await appendRows(api, dataset, rows, flags) : null;
52
+ print({ dataset: { id: dataset.id, name: dataset.name }, ...(source ? { source: source.source, run: source.feed } : {}), ...(added ? { rows: added } : {}) }, flags);
50
53
  }
51
54
 
52
55
  /** trodo datasets add DATASET --kind KIND <filter flags> [--live] [--include-trace]: another source. */
@@ -68,6 +71,11 @@ async function append([ref], flags) {
68
71
  const api = client(config.resolve(flags));
69
72
  const d = await resolveDataset(api, ref);
70
73
  const rows = await readRows(flags.file);
74
+ print({ dataset: d.name, ...(await appendRows(api, d, rows, flags)) }, flags);
75
+ }
76
+
77
+ /** Rows (plain or references to traffic) into dataset `d`; returns { written, skipped }. */
78
+ async function appendRows(api, d, rows, flags) {
71
79
  const REF = new Set(['run_id', 'span_id', 'conversation_id', 'eval_run_id']);
72
80
  const refs = { run_ids: [], span_ids: [], conversation_ids: [], eval_run_ids: [] };
73
81
  const plain = [];
@@ -97,7 +105,7 @@ async function append([ref], flags) {
97
105
  const out = await api.post(`/datasets/${d.id}/items`, { items: part });
98
106
  written += Number(out.written ?? out.added ?? (Array.isArray(out.items) ? out.items.length : part.length)); skipped += Number(out.skipped ?? 0);
99
107
  }
100
- print({ dataset: d.name, written, skipped }, flags);
108
+ return { written, skipped };
101
109
  }
102
110
 
103
111
  /**
@@ -171,4 +179,4 @@ async function versions([ref], flags) {
171
179
  print(out.versions || out, flags, [['version_no', 'v', 4], ['name', 'Name', 40], ['item_count', 'Rows', 8], ['created_at', 'At', 24]]);
172
180
  }
173
181
 
174
- module.exports = { list: listDatasets, get: getDataset, create, add: addSource, append, update, export: exportDataset, rows: exportDataset, versions, resolveDataset, UUID };
182
+ module.exports = { list: listDatasets, get: getDataset, create, add: addSource, append, update, export: exportDataset, rows: exportDataset, download: exportDataset, versions, resolveDataset, UUID };
@@ -3,6 +3,7 @@
3
3
  const config = require('../config');
4
4
  const { client } = require('../http');
5
5
  const { print } = require('../output');
6
+ const { usage, pick, int, unitVerdicts } = require('../common');
6
7
 
7
8
  /** trodo evals list: the names --eval takes, and each eval's steps (STEP.FIELD). */
8
9
  async function listEvals(_args, flags) {
@@ -14,4 +15,124 @@ async function listEvals(_args, flags) {
14
15
  print(flags.table ? rows : evals, flags, [['name', 'Eval', 44], ['target', 'Target', 12], ['enabled', 'On', 5], ['step_fields', 'Steps (STEP.FIELD)', 70]]);
15
16
  }
16
17
 
17
- module.exports = { list: listEvals };
18
+ async function resolveEval(api, ref) {
19
+ const { evals } = await api.get('/evals');
20
+ return pick(evals || [], ref, { kind: 'eval', listCmd: 'trodo evals list' });
21
+ }
22
+
23
+ /** trodo evals get EVAL: the eval, its production version's graph and its 30-day stats. */
24
+ async function getEval([ref], flags) {
25
+ const api = client(config.resolve(flags));
26
+ const e = await resolveEval(api, ref);
27
+ const [lineage, stats] = await Promise.all([
28
+ api.get(`/eval3/definitions/${e.id}`),
29
+ api.get(`/eval3/definitions/${e.id}/stats`).catch(() => null),
30
+ ]);
31
+ print({ ...lineage, steps: e.steps, stats: stats || undefined }, flags);
32
+ }
33
+
34
+ /** "7d" / "48h" / "30" → whole days for the verdict list (1-365). */
35
+ function daysOf(since) {
36
+ if (since == null || since === true) return undefined;
37
+ const m = /^(\d+)\s*([hd]?)$/.exec(String(since).trim());
38
+ if (!m) throw usage('--since takes days or hours here: 7d, 48h, 30.');
39
+ const n = Number(m[1]);
40
+ return Math.min(365, Math.max(1, m[2] === 'h' ? Math.ceil(n / 24) : n));
41
+ }
42
+
43
+ /**
44
+ * trodo evals verdicts EVAL [--failed | --passed] [--since 7d] [--version N] [--limit 50] [--offset N]
45
+ * trodo evals verdicts --run RUN_ID | --span SPAN_ID | --conversation ID
46
+ * The eval's verdicts, newest first; or every eval's newest verdict on one unit.
47
+ */
48
+ async function verdicts([ref], flags) {
49
+ const api = client(config.resolve(flags));
50
+ const unit = ['run', 'span', 'conversation'].find((k) => flags[k] && flags[k] !== true);
51
+ const cols = [['id', 'Verdict', 36], ['lineage_name', 'Eval', 34], ['passed', 'Passed', 6], ['score', 'Score', 6], ['reason_code', 'Reason', 22], ['created_at', 'At', 24]];
52
+ if (unit) {
53
+ const out = await api.get(`/eval3/units/${unit}/${encodeURIComponent(String(flags[unit]))}/verdicts`);
54
+ print(unitVerdicts(out), flags, cols);
55
+ return;
56
+ }
57
+ if (flags.failed && flags.passed) throw usage('Give --failed or --passed, not both.');
58
+ const e = await resolveEval(api, ref);
59
+ const q = {
60
+ limit: int(flags.limit, 50, 200),
61
+ offset: flags.offset,
62
+ days: daysOf(flags.since),
63
+ version: flags.version,
64
+ ...(flags.failed ? { passed: 'false' } : flags.passed ? { passed: 'true' } : {}),
65
+ };
66
+ const out = await api.get(`/eval3/definitions/${e.id}/runs`, q);
67
+ const rows = (out.runs || out.verdicts || []).map((r) => ({ lineage_name: e.name, ...r }));
68
+ print(flags.table ? rows : { eval: e.name, ...out }, flags, cols);
69
+ }
70
+
71
+ /** trodo evals verdict VERDICT_ID: one verdict, every step it took and why. */
72
+ async function verdict([id], flags) {
73
+ if (!id) throw usage('Usage: trodo evals verdict VERDICT_ID');
74
+ print(await client(config.resolve(flags)).get(`/eval3/runs/${encodeURIComponent(id)}`), flags);
75
+ }
76
+
77
+ /**
78
+ * trodo evals mark VERDICT_ID --right
79
+ * trodo evals mark VERDICT_ID --wrong --should pass|fail --reason TEXT
80
+ * A person's thumbs on a verdict. A thumbs-down says what it should have been and
81
+ * why; it becomes that verdict row's Expected and feeds the eval's improvement.
82
+ */
83
+ async function mark([id], flags) {
84
+ if (!id) throw usage('Usage: trodo evals mark VERDICT_ID --right | --wrong --should pass|fail --reason TEXT');
85
+ if (Boolean(flags.right) === Boolean(flags.wrong)) throw usage('Give --right or --wrong.');
86
+ const body = { source: 'v3', eval_run_id: String(id), thumbs: flags.right ? 'up' : 'down' };
87
+ if (flags.wrong) {
88
+ const should = String(flags.should || '');
89
+ if (!['pass', 'fail'].includes(should)) throw usage('--wrong needs --should pass or --should fail.');
90
+ if (!flags.reason || flags.reason === true) throw usage('--wrong needs --reason "why".');
91
+ body.corrected_passed = should === 'pass';
92
+ body.reason_text = String(flags.reason);
93
+ } else if (flags.reason && flags.reason !== true) body.reason_text = String(flags.reason);
94
+ print(await client(config.resolve(flags)).post('/eval3/feedback', body), flags);
95
+ }
96
+
97
+ /**
98
+ * trodo evals queue [--eval EVAL] [--graded] [--limit 50]
99
+ * Verdicts waiting on a person: an eval's human step stops there until someone
100
+ * picks one of its choices.
101
+ */
102
+ async function queue(_a, flags) {
103
+ const api = client(config.resolve(flags));
104
+ const lineageId = flags.eval && flags.eval !== true ? (await resolveEval(api, flags.eval)).id : undefined;
105
+ const out = await api.get('/eval3/queue', { status: flags.graded ? 'graded' : 'open', lineage_id: lineageId, limit: int(flags.limit, 50, 200), offset: flags.offset });
106
+ print(out.items || out, flags, [['id', 'Item', 36], ['lineage_name', 'Eval', 34], ['node_id', 'Step', 22], ['status', 'Status', 8], ['created_at', 'Waiting since', 24]]);
107
+ }
108
+
109
+ /** trodo evals review ITEM_ID: one waiting item with the path the eval took to it. */
110
+ async function review([id], flags) {
111
+ if (!id) throw usage('Usage: trodo evals review ITEM_ID');
112
+ print(await client(config.resolve(flags)).get(`/eval3/queue/${encodeURIComponent(id)}`), flags);
113
+ }
114
+
115
+ /** trodo evals grade ITEM_ID CHOICE: answer the human step; the eval carries on from there. */
116
+ async function grade([id, choice], flags) {
117
+ if (!id || !choice) throw usage('Usage: trodo evals grade ITEM_ID CHOICE (`trodo evals review ITEM_ID` lists the choices)');
118
+ print(await client(config.resolve(flags)).post(`/eval3/queue/${encodeURIComponent(id)}/grade`, { choice: String(choice) }), flags);
119
+ }
120
+
121
+ /**
122
+ * trodo evals steps EVAL [--since 30d] [--version N]
123
+ * The paths verdicts took through the eval's steps (its nodes), each with how
124
+ * many runs took it and how they came out: which step decides most failures.
125
+ */
126
+ async function steps([ref], flags) {
127
+ const api = client(config.resolve(flags));
128
+ const e = await resolveEval(api, ref);
129
+ const out = await api.get(`/eval3/definitions/${e.id}/steps`, { days: daysOf(flags.since) || 30, version: flags.version });
130
+ if (flags.table) {
131
+ const rows = (out.paths || []).map((p) => ({ path: (p.names || p.path || []).join(' → '), runs: p.runs, share: p.share, passed: p.passed, failed: p.failed, errored: p.errored }));
132
+ print(rows, flags, [['path', 'Path through the steps', 90], ['runs', 'Runs', 6], ['share', 'Share', 6], ['passed', 'Passed', 6], ['failed', 'Failed', 6], ['errored', 'Errored', 7]]);
133
+ return;
134
+ }
135
+ print({ eval: e.name, steps: e.steps, ...out }, flags);
136
+ }
137
+
138
+ module.exports = { list: listEvals, get: getEval, verdicts, verdict, mark, queue, review, grade, steps, daysOf };
@@ -0,0 +1,40 @@
1
+ 'use strict';
2
+
3
+ const config = require('../config');
4
+ const { client } = require('../http');
5
+ const { print } = require('../output');
6
+ const { usage, int } = require('../common');
7
+
8
+ const csv = (v) => (v === undefined || v === true ? undefined : [].concat(v).join(','));
9
+
10
+ /** trodo issues list [--status open,regression,closed] [--agent NAME] [--search TEXT] [--limit 50] */
11
+ async function list(_a, flags) {
12
+ const out = await client(config.resolve(flags)).get('/issues', {
13
+ status: csv(flags.status), agent: csv(flags.agent), search: flags.search !== true ? flags.search : undefined,
14
+ limit: int(flags.limit, 50, 200), offset: flags.offset,
15
+ });
16
+ print(out.issues || out, flags, [['id', 'Id', 6], ['status', 'Status', 10], ['agent_name', 'Agent', 28], ['title', 'Issue', 60], ['evidence_count', 'Evidence', 8], ['last_seen_at', 'Last seen', 24]]);
17
+ }
18
+
19
+ /** trodo issues get ID: the issue, its evidence (runs, spans, conversations) and its timeline. */
20
+ async function get([id], flags) {
21
+ if (!id) throw usage('Usage: trodo issues get ISSUE_ID');
22
+ print(await client(config.resolve(flags)).get(`/issues/${encodeURIComponent(id)}`), flags);
23
+ }
24
+
25
+ /** trodo issues failures ID: the recorded inputs of every failure, to replay against a fix. */
26
+ async function failures([id], flags) {
27
+ if (!id) throw usage('Usage: trodo issues failures ISSUE_ID');
28
+ print(await client(config.resolve(flags)).get(`/issues/${encodeURIComponent(id)}/failures`, { limit: flags.limit, cursor: flags.cursor !== true ? flags.cursor : undefined }), flags);
29
+ }
30
+
31
+ async function setStatus(id, status, flags) {
32
+ if (!id) throw usage(`Usage: trodo issues ${status === 'closed' ? 'close' : 'reopen'} ISSUE_ID`);
33
+ print(await client(config.resolve(flags)).post(`/issues/${encodeURIComponent(id)}/status`, { status }), flags);
34
+ }
35
+
36
+ /** trodo issues close ID: closed; a new failure of the same kind reopens it as a regression. */
37
+ const close = ([id], flags) => setStatus(id, 'closed', flags);
38
+ const reopen = ([id], flags) => setStatus(id, 'open', flags);
39
+
40
+ module.exports = { list, get, failures, close, reopen };
@@ -0,0 +1,125 @@
1
+ 'use strict';
2
+
3
+ const fs = require('fs');
4
+ const config = require('../config');
5
+ const { client } = require('../http');
6
+ const { print } = require('../output');
7
+ const { usage, pick } = require('../common');
8
+
9
+ /** What a prompt version is, without who saved it: what `pull` writes and `push` reads. */
10
+ const CONTENT_KEYS = ['messages', 'model', 'tools', 'response_format', 'variables'];
11
+
12
+ /** A refusal to edit, said as what to do about it. */
13
+ async function editing(prompt, fn) {
14
+ try { return await fn(); } catch (e) {
15
+ if (e && e.status === 403 && e.body && e.body.error === 'forbidden_edit') {
16
+ throw Object.assign(new Error(`You can't change "${prompt}": only its owner and the editors they add can. Ask the owner to add you (prompt page → Editors).`), { exitCode: 1 });
17
+ }
18
+ throw e;
19
+ }
20
+ }
21
+
22
+ async function resolvePrompt(api, ref) {
23
+ const { prompts } = await api.get('/prompts');
24
+ const hit = pick(prompts || [], ref, { kind: 'prompt', listCmd: 'trodo prompts list' });
25
+ const { prompt } = await api.get(`/prompts/${hit.id}`);
26
+ return prompt;
27
+ }
28
+
29
+ /** The version a --version N or --label L names; the newest one otherwise. */
30
+ function versionOf(prompt, flags) {
31
+ const versions = prompt.versions || [];
32
+ if (flags.version !== undefined && flags.version !== true) {
33
+ const v = versions.find((x) => String(x.version_no) === String(flags.version));
34
+ if (!v) throw usage(`"${prompt.name}" has no version ${flags.version}. \`trodo prompts versions "${prompt.name}"\` lists them.`);
35
+ return v;
36
+ }
37
+ if (flags.label !== undefined && flags.label !== true) {
38
+ const v = versions.find((x) => (x.labels || []).includes(String(flags.label)));
39
+ if (!v) throw usage(`No version of "${prompt.name}" has the label "${flags.label}".`);
40
+ return v;
41
+ }
42
+ return versions.reduce((a, b) => (!a || Number(b.version_no) > Number(a.version_no) ? b : a), null);
43
+ }
44
+
45
+ async function list(_a, flags) {
46
+ const { prompts } = await client(config.resolve(flags)).get('/prompts', { label: flags.label !== true ? flags.label : undefined, tag: flags.tag !== true ? flags.tag : undefined });
47
+ print(prompts || [], flags, [['name', 'Prompt', 36], ['current_version', 'Version', 8], ['labels', 'Labels', 24], ['id', 'Id', 36], ['updated_at', 'Updated', 24]]);
48
+ }
49
+
50
+ /** trodo prompts get PROMPT [--version N | --label production]: one version, whole. */
51
+ async function get([ref], flags) {
52
+ const prompt = await resolvePrompt(client(config.resolve(flags)), ref);
53
+ const v = versionOf(prompt, flags);
54
+ print({ id: prompt.id, name: prompt.name, description: prompt.description, tags: prompt.tags, version: v }, flags);
55
+ }
56
+
57
+ async function versions([ref], flags) {
58
+ const prompt = await resolvePrompt(client(config.resolve(flags)), ref);
59
+ const rows = (prompt.versions || []).map((v) => ({ version_no: v.version_no, labels: v.labels, commit_message: v.commit_message, created_by: v.created_by_email, created_at: v.created_at, content_hash: v.content_hash }));
60
+ print(rows, flags, [['version_no', 'v', 4], ['labels', 'Labels', 22], ['commit_message', 'Message', 44], ['created_by', 'By', 24], ['created_at', 'At', 24]]);
61
+ }
62
+
63
+ /** trodo prompts pull PROMPT [--version N | --label L] [-o prompt.json]: the content to edit and push back. */
64
+ async function pull([ref], flags) {
65
+ const prompt = await resolvePrompt(client(config.resolve(flags)), ref);
66
+ const v = versionOf(prompt, flags);
67
+ const out = { name: prompt.name, version_no: v.version_no };
68
+ for (const k of CONTENT_KEYS) out[k] = v[k];
69
+ if (flags.out && flags.out !== true) {
70
+ fs.writeFileSync(String(flags.out), `${JSON.stringify(out, null, 2)}\n`);
71
+ process.stderr.write(`${prompt.name} v${v.version_no} → ${flags.out}\n`);
72
+ return;
73
+ }
74
+ print(out, flags);
75
+ }
76
+
77
+ /**
78
+ * trodo prompts push PROMPT -f prompt.json [--message "why"] [--label staging]
79
+ * A new version from a file `pull` wrote (or any { messages, model?, tools?,
80
+ * variables? }). The prompt is made when no prompt has that name. Content equal
81
+ * to the newest version cuts no version.
82
+ */
83
+ async function push([ref], flags) {
84
+ const file = flags.file;
85
+ if (!file || file === true) throw usage('Usage: trodo prompts push PROMPT -f prompt.json [--message "why"] [--label staging]');
86
+ let content;
87
+ try { content = JSON.parse(fs.readFileSync(String(file), 'utf8')); } catch (e) { throw usage(`${file} is not a JSON prompt: ${e.message}`); }
88
+ if (!Array.isArray(content.messages) || !content.messages.length) throw usage(`${file} has no messages.`);
89
+ const api = client(config.resolve(flags));
90
+ const name = ref && ref !== true ? String(ref) : content.name;
91
+ if (!name) throw usage('Name the prompt.');
92
+ const { prompts } = await api.get('/prompts');
93
+ let target = (prompts || []).find((p) => p.id === name || p.name === name);
94
+ let created = false;
95
+ if (!target) {
96
+ const made = await api.post('/prompts', { name, ...(content.description ? { description: content.description } : {}) });
97
+ target = made.prompt || made;
98
+ created = true;
99
+ }
100
+ const body = { messages: content.messages };
101
+ for (const k of ['model', 'tools', 'response_format', 'variables']) if (content[k] !== undefined) body[k] = content[k];
102
+ if (flags.message && flags.message !== true) body.commit_message = String(flags.message);
103
+ if (flags.label && flags.label !== true) body.labels = [].concat(flags.label).map(String);
104
+ const out = await editing(name, () => api.post(`/prompts/${target.id}/versions`, body));
105
+ const v = out.version || {};
106
+ print({ prompt: name, id: target.id, created, version_no: v.version_no, labels: v.labels, unchanged: Boolean(out.unchanged) }, flags);
107
+ }
108
+
109
+ /**
110
+ * trodo prompts label PROMPT VERSION LABEL [--remove]
111
+ * Moves LABEL (say production) to VERSION. Agents that fetch the prompt by that
112
+ * label run the new version from their next call.
113
+ */
114
+ async function label([ref, versionNo, name], flags) {
115
+ if (!ref || !versionNo || !name) throw usage('Usage: trodo prompts label PROMPT VERSION LABEL [--remove]');
116
+ const api = client(config.resolve(flags));
117
+ const prompt = await resolvePrompt(api, ref);
118
+ const v = versionOf(prompt, { version: versionNo });
119
+ const out = await editing(prompt.name, () => (flags.remove
120
+ ? api.del(`/prompts/${prompt.id}/versions/${v.version_no}/label/${encodeURIComponent(String(name))}`)
121
+ : api.post(`/prompts/${prompt.id}/versions/${v.version_no}/label`, { label: String(name) })));
122
+ print({ prompt: prompt.name, version_no: v.version_no, ...(out.version ? { labels: out.version.labels } : out) }, flags);
123
+ }
124
+
125
+ module.exports = { list, get, versions, pull, push, label };