trodo-cli 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -2
- package/bin/trodo.js +7 -0
- package/lib/args.js +45 -0
- package/lib/commands/agents.js +56 -0
- package/lib/commands/auth.js +107 -0
- package/lib/commands/datasets.js +174 -0
- package/lib/commands/evals.js +17 -0
- package/lib/commands/runs.js +41 -0
- package/lib/commands/traces.js +46 -0
- package/lib/config.js +37 -0
- package/lib/filters.js +157 -0
- package/lib/http.js +73 -0
- package/lib/main.js +96 -0
- package/lib/output.js +50 -0
- package/package.json +28 -4
package/README.md
CHANGED
|
@@ -1,3 +1,25 @@
|
|
|
1
|
-
#
|
|
1
|
+
# trodo-cli
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Trodo from the terminal, a script, CI or a coding agent: pull traces by what
|
|
4
|
+
your evals said about them, keep whole runs with their span trees, build and
|
|
5
|
+
label datasets, run agents, and export fine-tuning files.
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
npm i -g trodo-cli
|
|
9
|
+
trodo login # opens the app to connect this terminal
|
|
10
|
+
trodo evals list --table
|
|
11
|
+
trodo traces count --kind runs --since 30d --eval "any:passed" --eval "any:not-failed"
|
|
12
|
+
trodo traces export --kind spans --since 30d --eval any:passed --format openai --target output -o sft.jsonl
|
|
13
|
+
trodo datasets create "Good refunds" --kind runs --eval "Refund policy:passed" --include-trace
|
|
14
|
+
trodo datasets export "Good refunds" --rows passed --format openai --target output -o train.jsonl
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
`trodo help` lists every command and filter. Output is JSON (add `--table`
|
|
18
|
+
for a table). Exit codes: 0 ok, 1 API error, 2 usage.
|
|
19
|
+
|
|
20
|
+
Auth: `trodo login` (browser), `trodo login --key trodo_mk_…`, or
|
|
21
|
+
`TRODO_API_KEY` / `TRODO_API_URL`. Keys live in Settings → MCP; one acts as
|
|
22
|
+
the person who made it, on its team, with its scopes.
|
|
23
|
+
|
|
24
|
+
Coding agents: the `trodo-datasets` skill (skills/trodo-datasets/SKILL.md in
|
|
25
|
+
the Trodo repo) teaches the whole loop.
|
package/bin/trodo.js
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
// `trodo … | head` closes the pipe early: that is done, not an error.
|
|
5
|
+
process.stdout.on('error', (e) => { if (e && e.code === 'EPIPE') process.exit(0); throw e; });
|
|
6
|
+
|
|
7
|
+
require('../lib/main').main(process.argv.slice(2)).then((code) => { process.exitCode = code; });
|
package/lib/args.js
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Flags: `--name value`, `--name=value`, `--flag` (true), `-o file`, `-f file`.
|
|
5
|
+
* A flag given twice collects into an array (`--where a --where b`).
|
|
6
|
+
* `--` ends flags. Everything else is positional.
|
|
7
|
+
*/
|
|
8
|
+
const SHORT = { o: 'out', f: 'file', h: 'help', n: 'limit' };
|
|
9
|
+
const BOOLEAN = new Set(['help', 'json', 'table', 'live', 'include-trace', 'wait', 'yes', 'no-trace', 'no-browser', 'any-span']);
|
|
10
|
+
|
|
11
|
+
function parse(argv) {
|
|
12
|
+
const flags = {};
|
|
13
|
+
const positional = [];
|
|
14
|
+
const put = (k, v) => {
|
|
15
|
+
if (flags[k] === undefined) flags[k] = v;
|
|
16
|
+
else flags[k] = [].concat(flags[k], v);
|
|
17
|
+
};
|
|
18
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
19
|
+
const a = argv[i];
|
|
20
|
+
if (a === '--') { positional.push(...argv.slice(i + 1)); break; }
|
|
21
|
+
if (a.startsWith('--')) {
|
|
22
|
+
const eq = a.indexOf('=');
|
|
23
|
+
const key = eq > 0 ? a.slice(2, eq) : a.slice(2);
|
|
24
|
+
if (eq > 0) { put(key, a.slice(eq + 1)); continue; }
|
|
25
|
+
if (BOOLEAN.has(key)) { put(key, true); continue; }
|
|
26
|
+
const next = argv[i + 1];
|
|
27
|
+
if (next === undefined || (next.startsWith('--') && next.length > 2)) { put(key, true); continue; }
|
|
28
|
+
put(key, next); i += 1;
|
|
29
|
+
continue;
|
|
30
|
+
}
|
|
31
|
+
if (/^-[a-zA-Z]$/.test(a)) {
|
|
32
|
+
const key = SHORT[a[1]] || a[1];
|
|
33
|
+
if (BOOLEAN.has(key)) { put(key, true); continue; }
|
|
34
|
+
put(key, argv[i + 1]); i += 1;
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
positional.push(a);
|
|
38
|
+
}
|
|
39
|
+
return { flags, positional };
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** A flag as a list (absent → []). */
|
|
43
|
+
const list = (v) => (v === undefined || v === true ? [] : [].concat(v)).map(String);
|
|
44
|
+
|
|
45
|
+
module.exports = { parse, list };
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const config = require('../config');
|
|
4
|
+
const { client } = require('../http');
|
|
5
|
+
const { print } = require('../output');
|
|
6
|
+
|
|
7
|
+
const usage = (msg) => { const e = new Error(msg); e.exitCode = 2; return e; };
|
|
8
|
+
const DONE = new Set(['succeeded', 'failed', 'cancelled', 'canceled', 'stopped']);
|
|
9
|
+
|
|
10
|
+
async function resolveAgent(api, ref) {
|
|
11
|
+
if (!ref) throw usage('Name the agent (its name or id).');
|
|
12
|
+
const { workflows } = await api.get('/agents');
|
|
13
|
+
const all = workflows || [];
|
|
14
|
+
const hit = all.find((w) => w.id === ref) || all.find((w) => w.name === ref) || all.find((w) => w.name.toLowerCase() === String(ref).toLowerCase());
|
|
15
|
+
if (!hit) throw usage(`No agent "${ref}". \`trodo agents list\` shows them.`);
|
|
16
|
+
return hit;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
async function listAgents(_a, flags) {
|
|
20
|
+
const { workflows } = await client(config.resolve(flags)).get('/agents');
|
|
21
|
+
print(workflows || [], flags, [['name', 'Agent', 44], ['status', 'Status', 10], ['id', 'Id', 36], ['updated_at', 'Updated', 24]]);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** trodo agents run AGENT [--input JSON] [--wait]: start its published version, as Run manually does. */
|
|
25
|
+
async function run([ref], flags) {
|
|
26
|
+
const api = client(config.resolve(flags));
|
|
27
|
+
const a = await resolveAgent(api, ref);
|
|
28
|
+
let input = null;
|
|
29
|
+
if (flags.input) { try { input = JSON.parse(String(flags.input)); } catch (e) { throw usage(`--input is not JSON: ${e.message}`); } }
|
|
30
|
+
const { execution } = await api.post(`/agents/${a.id}/executions`, { input });
|
|
31
|
+
if (!flags.wait) return print({ agent: a.name, execution }, flags);
|
|
32
|
+
let exec = execution;
|
|
33
|
+
const deadline = Date.now() + (Number(flags.timeout) || 900) * 1000;
|
|
34
|
+
while (!DONE.has(String(exec.status)) && Date.now() < deadline) {
|
|
35
|
+
// eslint-disable-next-line no-await-in-loop
|
|
36
|
+
await new Promise((r) => setTimeout(r, 2000));
|
|
37
|
+
// eslint-disable-next-line no-await-in-loop
|
|
38
|
+
exec = (await api.get(`/agents/executions/${exec.id}`, { lite: 1 })).execution;
|
|
39
|
+
}
|
|
40
|
+
print({ agent: a.name, execution: exec }, flags);
|
|
41
|
+
if (exec.status === 'failed') process.exitCode = 1;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
async function executions([ref], flags) {
|
|
45
|
+
const api = client(config.resolve(flags));
|
|
46
|
+
const a = await resolveAgent(api, ref);
|
|
47
|
+
const out = await api.get(`/agents/${a.id}/executions`, { limit: flags.limit || 20 });
|
|
48
|
+
print(out.executions || out, flags, [['id', 'Execution', 36], ['status', 'Status', 10], ['started_at', 'Started', 24], ['finished_at', 'Finished', 24]]);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async function execution([id], flags) {
|
|
52
|
+
if (!id) throw usage('Usage: trodo agents execution EXECUTION_ID');
|
|
53
|
+
print(await client(config.resolve(flags)).get(`/agents/executions/${encodeURIComponent(id)}`), flags);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
module.exports = { list: listAgents, run, executions, execution };
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const config = require('../config');
|
|
4
|
+
const { client } = require('../http');
|
|
5
|
+
const { print } = require('../output');
|
|
6
|
+
|
|
7
|
+
async function readHidden(prompt) {
|
|
8
|
+
if (!process.stdin.isTTY) {
|
|
9
|
+
const chunks = [];
|
|
10
|
+
for await (const c of process.stdin) chunks.push(c);
|
|
11
|
+
return Buffer.concat(chunks).toString('utf8').trim();
|
|
12
|
+
}
|
|
13
|
+
process.stdout.write(prompt);
|
|
14
|
+
return new Promise((resolve) => {
|
|
15
|
+
let v = '';
|
|
16
|
+
process.stdin.setRawMode(true); process.stdin.resume();
|
|
17
|
+
const onData = (b) => {
|
|
18
|
+
const s = b.toString('utf8');
|
|
19
|
+
if (s === '\r' || s === '\n' || s === '\u0004') { process.stdin.setRawMode(false); process.stdin.pause(); process.stdin.off('data', onData); process.stdout.write('\n'); resolve(v.trim()); }
|
|
20
|
+
else if (s === '\u0003') { process.stdout.write('\n'); process.exit(130); }
|
|
21
|
+
else if (s === '\u007f') v = v.slice(0, -1);
|
|
22
|
+
else v += s;
|
|
23
|
+
};
|
|
24
|
+
process.stdin.on('data', onData);
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Open a URL in the default browser; false when there is none to open. */
|
|
29
|
+
function openBrowser(url) {
|
|
30
|
+
// eslint-disable-next-line global-require
|
|
31
|
+
const { spawn } = require('child_process');
|
|
32
|
+
const cmd = process.platform === 'darwin' ? ['open', [url]] : process.platform === 'win32' ? ['cmd', ['/c', 'start', '', url]] : ['xdg-open', [url]];
|
|
33
|
+
try {
|
|
34
|
+
const p = spawn(cmd[0], cmd[1], { stdio: 'ignore', detached: true });
|
|
35
|
+
p.on('error', () => {});
|
|
36
|
+
p.unref();
|
|
37
|
+
return true;
|
|
38
|
+
} catch { return false; }
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
42
|
+
|
|
43
|
+
/** The browser sign-in: a code here, Connect in the app, a key back. */
|
|
44
|
+
async function browserLogin(url, flags) {
|
|
45
|
+
// eslint-disable-next-line global-require
|
|
46
|
+
const os = require('os');
|
|
47
|
+
// eslint-disable-next-line global-require
|
|
48
|
+
const pkg = require('../../package.json');
|
|
49
|
+
const fetchImpl = globalThis.fetch;
|
|
50
|
+
const post = async (p, body) => {
|
|
51
|
+
const res = await fetchImpl(`${url}/api/v1${p}`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'User-Agent': `trodo-cli/${pkg.version}` }, body: JSON.stringify(body) });
|
|
52
|
+
const data = await res.json().catch(() => ({}));
|
|
53
|
+
if (!res.ok) { const e = new Error(data.message || data.error || `HTTP ${res.status} from ${url}`); e.status = res.status; throw e; }
|
|
54
|
+
return data;
|
|
55
|
+
};
|
|
56
|
+
const start = await post('/auth/device', { host: os.hostname().replace(/\.local$/, ''), client: `trodo-cli ${pkg.version}` });
|
|
57
|
+
process.stderr.write(`\nYour code: ${start.user_code}\n`);
|
|
58
|
+
const opened = !flags['no-browser'] && openBrowser(start.verification_url);
|
|
59
|
+
process.stderr.write(`${opened ? 'Opened' : 'Open'} ${start.verification_url}\n${opened ? 'Check the code there and press Connect.' : 'Sign in there, check the code and press Connect.'} Waiting…\n`);
|
|
60
|
+
const deadline = Date.now() + (start.expires_in || 600) * 1000;
|
|
61
|
+
while (Date.now() < deadline) {
|
|
62
|
+
// eslint-disable-next-line no-await-in-loop
|
|
63
|
+
await sleep((start.interval || 2) * 1000);
|
|
64
|
+
// eslint-disable-next-line no-await-in-loop
|
|
65
|
+
const out = await post('/auth/device/token', { device_code: start.device_code });
|
|
66
|
+
if (out.status === 'approved') return out.api_key;
|
|
67
|
+
if (out.status === 'denied') { const e = new Error('Sign-in was refused in the browser.'); e.exitCode = 1; throw e; }
|
|
68
|
+
if (out.status === 'expired') break;
|
|
69
|
+
}
|
|
70
|
+
const e = new Error('The sign-in code expired. Run `trodo login` again.');
|
|
71
|
+
e.exitCode = 1;
|
|
72
|
+
throw e;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* trodo login [--url URL] [--no-browser]: sign in through the app (a code, Connect, done).
|
|
77
|
+
* trodo login --key trodo_mk_…: use a key you made in Settings → MCP (CI, no browser).
|
|
78
|
+
* Saved to ~/.trodo/config.json, mode 600.
|
|
79
|
+
*/
|
|
80
|
+
async function login(_args, flags) {
|
|
81
|
+
const url = config.resolve(flags).url;
|
|
82
|
+
let key;
|
|
83
|
+
// --key K uses that key; --key alone (or --key -) reads it from the terminal or stdin.
|
|
84
|
+
if (flags.key === true || flags.key === '-') key = await readHidden('API key (trodo_mk_…, Settings → MCP): ');
|
|
85
|
+
else if (flags.key) key = String(flags.key);
|
|
86
|
+
else key = await browserLogin(url, flags);
|
|
87
|
+
if (!key.startsWith('trodo_mk_')) { const e = new Error('That is not a Trodo API key (they start with trodo_mk_).'); e.exitCode = 2; throw e; }
|
|
88
|
+
const me = await client({ url, key }).get('/me');
|
|
89
|
+
const cfg = config.read();
|
|
90
|
+
const file = config.write({ ...cfg, api_key: key, api_url: url });
|
|
91
|
+
process.stderr.write(`Signed in to ${me.team.name} as ${me.user.email}. Saved to ${file}.\n`);
|
|
92
|
+
print({ team: me.team, scopes: me.key.scopes }, flags);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
async function whoami(_args, flags) {
|
|
96
|
+
print(await client(config.resolve(flags)).get('/me'), flags);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
async function logout(_args, flags) {
|
|
100
|
+
const cfg = config.read();
|
|
101
|
+
delete cfg.api_key;
|
|
102
|
+
config.write(cfg);
|
|
103
|
+
process.stderr.write('Removed the saved key.\n');
|
|
104
|
+
print({ ok: true }, flags);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
module.exports = { login, whoami, logout };
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const config = require('../config');
|
|
4
|
+
const { client } = require('../http');
|
|
5
|
+
const { print, sink, readRows } = require('../output');
|
|
6
|
+
const { buildSpec } = require('./traces');
|
|
7
|
+
|
|
8
|
+
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
9
|
+
const usage = (msg) => { const e = new Error(msg); e.exitCode = 2; return e; };
|
|
10
|
+
|
|
11
|
+
async function resolveDataset(api, ref) {
|
|
12
|
+
if (!ref) throw usage('Name the dataset (its name or id).');
|
|
13
|
+
const { datasets } = await api.get('/datasets');
|
|
14
|
+
const all = datasets || [];
|
|
15
|
+
const hit = all.find((d) => d.id === ref) || all.find((d) => d.name === ref) || all.find((d) => d.name.toLowerCase() === String(ref).toLowerCase());
|
|
16
|
+
if (!hit) throw usage(`No dataset "${ref}". \`trodo datasets list\` shows them.`);
|
|
17
|
+
return hit;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
async function listDatasets(_a, flags) {
|
|
21
|
+
const { datasets } = await client(config.resolve(flags)).get('/datasets');
|
|
22
|
+
print(datasets || [], flags, [['name', 'Dataset', 40], ['item_count', 'Rows', 8], ['expected_count', 'With Expected', 14], ['id', 'Id', 36], ['updated_at', 'Updated', 24]]);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
async function getDataset([ref], flags) {
|
|
26
|
+
const api = client(config.resolve(flags));
|
|
27
|
+
const d = await resolveDataset(api, ref);
|
|
28
|
+
const [sources, versions] = await Promise.all([
|
|
29
|
+
api.get(`/datasets/${d.id}/sources`).catch(() => ({ sources: [] })),
|
|
30
|
+
api.get(`/datasets/${d.id}/versions`).catch(() => ({ versions: [] })),
|
|
31
|
+
]);
|
|
32
|
+
print({ ...d, sources: sources.sources || sources, versions: (versions.versions || versions || []).slice(0, 20) }, flags);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* trodo datasets create NAME [--description D]
|
|
37
|
+
* [--kind spans|runs|conversations|verdicts <filter flags> [--live] [--include-trace]]
|
|
38
|
+
* With --kind, the dataset starts with a source: what the filter matches is copied now
|
|
39
|
+
* (and, with --live, every 15 minutes after).
|
|
40
|
+
*/
|
|
41
|
+
async function create([name], flags) {
|
|
42
|
+
if (!name) throw usage('Usage: trodo datasets create NAME [--kind runs --eval "Grounded:passed" --include-trace]');
|
|
43
|
+
const api = client(config.resolve(flags));
|
|
44
|
+
// The filter is checked before anything is made: a typo never leaves an empty dataset behind.
|
|
45
|
+
const spec = flags.kind ? await buildSpec(api, flags, String(flags.kind)) : null;
|
|
46
|
+
const { dataset } = await api.post('/datasets', { name, ...(flags.description ? { description: String(flags.description) } : {}) });
|
|
47
|
+
let source = null;
|
|
48
|
+
if (spec) source = await api.post(`/datasets/${dataset.id}/sources`, { ...spec, live: Boolean(flags.live) });
|
|
49
|
+
print({ dataset: { id: dataset.id, name: dataset.name }, ...(source ? { source: source.source, run: source.feed } : {}) }, flags);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** trodo datasets add DATASET --kind KIND <filter flags> [--live] [--include-trace]: another source. */
|
|
53
|
+
async function addSource([ref], flags) {
|
|
54
|
+
const api = client(config.resolve(flags));
|
|
55
|
+
const d = await resolveDataset(api, ref);
|
|
56
|
+
const spec = await buildSpec(api, flags, String(flags.kind || 'runs'));
|
|
57
|
+
const out = await api.post(`/datasets/${d.id}/sources`, { ...spec, live: Boolean(flags.live) });
|
|
58
|
+
print({ dataset: d.name, source: out.source, run: out.feed }, flags);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* trodo datasets append DATASET [-f rows.jsonl | stdin] [--include-trace]
|
|
63
|
+
* A line is a plain row { input | messages, output?, expected?, metadata? } or a
|
|
64
|
+
* reference to traffic ({ run_id } / { span_id } / { conversation_id } / { eval_run_id }),
|
|
65
|
+
* which the server fills with Input, Output and the evaluations. Rows already held are skipped.
|
|
66
|
+
*/
|
|
67
|
+
async function append([ref], flags) {
|
|
68
|
+
const api = client(config.resolve(flags));
|
|
69
|
+
const d = await resolveDataset(api, ref);
|
|
70
|
+
const rows = await readRows(flags.file);
|
|
71
|
+
const REF = new Set(['run_id', 'span_id', 'conversation_id', 'eval_run_id']);
|
|
72
|
+
const refs = { run_ids: [], span_ids: [], conversation_ids: [], eval_run_ids: [] };
|
|
73
|
+
const plain = [];
|
|
74
|
+
for (const r of rows) {
|
|
75
|
+
const keys = r && typeof r === 'object' ? Object.keys(r) : [];
|
|
76
|
+
if (keys.length && keys.every((k) => REF.has(k))) {
|
|
77
|
+
if (r.eval_run_id) refs.eval_run_ids.push(String(r.eval_run_id));
|
|
78
|
+
else if (r.span_id) refs.span_ids.push(String(r.span_id));
|
|
79
|
+
else if (r.run_id) refs.run_ids.push(String(r.run_id));
|
|
80
|
+
else refs.conversation_ids.push(String(r.conversation_id));
|
|
81
|
+
} else if (r && typeof r === 'object') {
|
|
82
|
+
const { expected, ...rest } = r;
|
|
83
|
+
plain.push(expected !== undefined && rest.expected_output === undefined ? { ...rest, expected_output: expected } : rest);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
let written = 0; let skipped = 0;
|
|
87
|
+
const chunk = (arr, n) => Array.from({ length: Math.ceil(arr.length / n) }, (_, i) => arr.slice(i * n, i * n + n));
|
|
88
|
+
for (const [key, ids] of Object.entries(refs)) {
|
|
89
|
+
for (const part of chunk(ids, 200)) {
|
|
90
|
+
// eslint-disable-next-line no-await-in-loop
|
|
91
|
+
const out = await api.post(`/datasets/${d.id}/items`, { [key]: part, ...(flags['include-trace'] ? { include_trace: true } : {}) });
|
|
92
|
+
written += Number(out.written ?? out.added ?? 0); skipped += Number(out.skipped ?? 0);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
for (const part of chunk(plain, 500)) {
|
|
96
|
+
// eslint-disable-next-line no-await-in-loop
|
|
97
|
+
const out = await api.post(`/datasets/${d.id}/items`, { items: part });
|
|
98
|
+
written += Number(out.written ?? out.added ?? (Array.isArray(out.items) ? out.items.length : part.length)); skipped += Number(out.skipped ?? 0);
|
|
99
|
+
}
|
|
100
|
+
print({ dataset: d.name, written, skipped }, flags);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* trodo datasets update DATASET [-f updates.jsonl | stdin] [--label person|derived] [--name "version name"]
|
|
105
|
+
* Lines: { id, expected?, output?, metadata? } — the row ids an export or `rows` gave you.
|
|
106
|
+
* One version for the whole call. Expected counts as a person's unless --label derived.
|
|
107
|
+
*/
|
|
108
|
+
async function update([ref], flags) {
|
|
109
|
+
const api = client(config.resolve(flags));
|
|
110
|
+
const d = await resolveDataset(api, ref);
|
|
111
|
+
const rows = (await readRows(flags.file)).filter((r) => r && r.id);
|
|
112
|
+
if (!rows.length) throw usage('No updates: give lines like {"id": "…", "expected": "…"}.');
|
|
113
|
+
let updated = 0; let unchanged = 0; const missing = []; let version = null;
|
|
114
|
+
for (let i = 0; i < rows.length; i += 5000) {
|
|
115
|
+
// eslint-disable-next-line no-await-in-loop
|
|
116
|
+
const out = await api.post(`/datasets/${d.id}/items/update`, {
|
|
117
|
+
updates: rows.slice(i, i + 5000),
|
|
118
|
+
label: flags.label === 'derived' ? 'derived' : 'person',
|
|
119
|
+
...(flags.name ? { version_name: String(flags.name) } : {}),
|
|
120
|
+
});
|
|
121
|
+
updated += out.updated; unchanged += out.unchanged; missing.push(...(out.missing || [])); version = out.version || version;
|
|
122
|
+
}
|
|
123
|
+
print({ dataset: d.name, updated, unchanged, missing: missing.slice(0, 50), version }, flags);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* trodo datasets export DATASET [--rows all|expected|no-expected|passed|failed|marked-wrong]
|
|
128
|
+
* [--eval NAME] [--min-score N] [--format trodo|openai|dpo] [--target expected|output|expected-else-output]
|
|
129
|
+
* [--version N] [-o file]
|
|
130
|
+
* `--rows passed --format openai --target output` is a supervised fine-tuning file
|
|
131
|
+
* made of what went well, no labeling needed.
|
|
132
|
+
*/
|
|
133
|
+
async function exportDataset([ref], flags) {
|
|
134
|
+
const api = client(config.resolve(flags));
|
|
135
|
+
const d = await resolveDataset(api, ref);
|
|
136
|
+
const query = {
|
|
137
|
+
rows: flags.rows ? String(flags.rows).replace(/-/g, '_') : undefined,
|
|
138
|
+
eval: flags.eval ? String(flags.eval) : undefined,
|
|
139
|
+
min_score: flags['min-score'],
|
|
140
|
+
format: flags.format ? String(flags.format) : undefined,
|
|
141
|
+
target: flags.target ? String(flags.target).replace(/-/g, '_') : undefined,
|
|
142
|
+
version: flags.version === true ? undefined : flags.version,
|
|
143
|
+
};
|
|
144
|
+
const res = await api.raw('GET', `/datasets/${d.id}/export`, { query });
|
|
145
|
+
if (!res.ok) {
|
|
146
|
+
const t = await res.text(); let body; try { body = JSON.parse(t); } catch { body = { message: t }; }
|
|
147
|
+
const e = new Error(body.message || body.error || `HTTP ${res.status}`); e.exitCode = 1; throw e;
|
|
148
|
+
}
|
|
149
|
+
const out = sink(flags.out && flags.out !== true ? String(flags.out) : null);
|
|
150
|
+
const decoder = new TextDecoder();
|
|
151
|
+
let buf = '';
|
|
152
|
+
for await (const c of res.body) {
|
|
153
|
+
buf += decoder.decode(c, { stream: true });
|
|
154
|
+
let i;
|
|
155
|
+
while ((i = buf.indexOf('\n')) >= 0) {
|
|
156
|
+
const line = buf.slice(0, i).trim(); buf = buf.slice(i + 1);
|
|
157
|
+
// eslint-disable-next-line no-await-in-loop
|
|
158
|
+
if (line) await out.write(JSON.parse(line));
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
if (buf.trim()) await out.write(JSON.parse(buf.trim()));
|
|
162
|
+
await out.close();
|
|
163
|
+
const n = res.headers.get('x-trodo-export-rows'); const skipped = res.headers.get('x-trodo-export-skipped');
|
|
164
|
+
process.stderr.write(`${n} rows from ${d.name}${skipped ? `, ${skipped} skipped (no input or no target)` : ''}${flags.out ? ` → ${flags.out}` : ''}\n`);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
async function versions([ref], flags) {
|
|
168
|
+
const api = client(config.resolve(flags));
|
|
169
|
+
const d = await resolveDataset(api, ref);
|
|
170
|
+
const out = await api.get(`/datasets/${d.id}/versions`);
|
|
171
|
+
print(out.versions || out, flags, [['version_no', 'v', 4], ['name', 'Name', 40], ['item_count', 'Rows', 8], ['created_at', 'At', 24]]);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
module.exports = { list: listDatasets, get: getDataset, create, add: addSource, append, update, export: exportDataset, rows: exportDataset, versions, resolveDataset, UUID };
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const config = require('../config');
|
|
4
|
+
const { client } = require('../http');
|
|
5
|
+
const { print } = require('../output');
|
|
6
|
+
|
|
7
|
+
/** trodo evals list: the names --eval takes, and each eval's steps (STEP.FIELD). */
|
|
8
|
+
async function listEvals(_args, flags) {
|
|
9
|
+
const { evals } = await client(config.resolve(flags)).get('/evals');
|
|
10
|
+
const rows = (evals || []).map((e) => ({
|
|
11
|
+
...e,
|
|
12
|
+
step_fields: (e.steps || []).flatMap((s) => (s.fields || []).map((f) => `${s.name}.${f.name} (${f.type === 'enum' ? (f.values || []).join('|') : f.type})`)).join('; '),
|
|
13
|
+
}));
|
|
14
|
+
print(flags.table ? rows : evals, flags, [['name', 'Eval', 44], ['target', 'Target', 12], ['enabled', 'On', 5], ['step_fields', 'Steps (STEP.FIELD)', 70]]);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
module.exports = { list: listEvals };
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const config = require('../config');
|
|
4
|
+
const { client } = require('../http');
|
|
5
|
+
const { print } = require('../output');
|
|
6
|
+
const { buildSpec } = require('./traces');
|
|
7
|
+
|
|
8
|
+
const DAY = 86400e3;
|
|
9
|
+
/** A filter's time → { time_start, time_end } for the runs list. */
|
|
10
|
+
function windowOf(time, now = Date.now()) {
|
|
11
|
+
if (time.from || time.to) return { time_start: time.from || new Date(now - 7 * DAY).toISOString(), time_end: time.to || new Date(now).toISOString() };
|
|
12
|
+
const p = String(time.preset || '7d');
|
|
13
|
+
const start = new Date(now); start.setHours(0, 0, 0, 0);
|
|
14
|
+
if (p === 'today') return { time_start: start.toISOString(), time_end: new Date(now).toISOString() };
|
|
15
|
+
if (p === 'yesterday') return { time_start: new Date(start.getTime() - DAY).toISOString(), time_end: new Date(start.getTime() - 1).toISOString() };
|
|
16
|
+
const m = /^(\d+)([mhd])$/.exec(p);
|
|
17
|
+
const n = m ? Number(m[1]) : 7;
|
|
18
|
+
const unit = !m ? DAY : m[2] === 'h' ? 3600e3 : m[2] === 'd' ? DAY : (['3m', '6m', '12m'].includes(p) ? 30 * DAY : 60e3);
|
|
19
|
+
return { time_start: new Date(now - n * unit).toISOString(), time_end: new Date(now).toISOString() };
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** trodo runs list [filter flags] [--limit 50] */
|
|
23
|
+
async function listRuns(_args, flags) {
|
|
24
|
+
const api = client(config.resolve(flags));
|
|
25
|
+
const spec = await buildSpec(api, flags, 'runs');
|
|
26
|
+
const mf = [...spec.filter.modularFilters];
|
|
27
|
+
if (spec.filter.eval_conditions) mf.push({ category: 'evals', field: 'eval_conditions', operator: 'is', value: spec.filter.eval_conditions });
|
|
28
|
+
const query = { ...windowOf(spec.filter.time), limit: spec.options.limit || 50 };
|
|
29
|
+
// The runs list decodes this once more (decodeURIComponent then JSON), as the app sends it.
|
|
30
|
+
if (mf.length) query.modularFilters = encodeURIComponent(JSON.stringify(mf));
|
|
31
|
+
const out = await api.get('/runs', query);
|
|
32
|
+
print(out.runs || out, flags, [['run_id', 'Run', 36], ['agent_name', 'Agent', 30], ['status', 'Status', 8], ['started_at', 'Started', 24], ['duration_ms', 'ms', 8]]);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** trodo runs get RUN_ID: the run and its spans. */
|
|
36
|
+
async function getRun([id], flags) {
|
|
37
|
+
if (!id) { const e = new Error('Usage: trodo runs get RUN_ID'); e.exitCode = 2; throw e; }
|
|
38
|
+
print(await client(config.resolve(flags)).get(`/runs/${encodeURIComponent(id)}`, { include: 'spans,messages' }), flags);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
module.exports = { list: listRuns, get: getRun, windowOf };
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const config = require('../config');
|
|
4
|
+
const { client } = require('../http');
|
|
5
|
+
const { print, sink } = require('../output');
|
|
6
|
+
const { specFromFlags, needsEvals } = require('../filters');
|
|
7
|
+
|
|
8
|
+
async function evalsIfNeeded(api, flags) {
|
|
9
|
+
return needsEvals(flags) ? (await api.get('/evals')).evals || [] : [];
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
async function buildSpec(api, flags, kind) {
|
|
13
|
+
return specFromFlags(flags, { kind, evals: await evalsIfNeeded(api, flags) });
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** trodo traces count [filter flags] */
|
|
17
|
+
async function count(_args, flags) {
|
|
18
|
+
const api = client(config.resolve(flags));
|
|
19
|
+
const spec = await buildSpec(api, flags, flags.kind || 'runs');
|
|
20
|
+
const out = await api.post('/traces/count', spec);
|
|
21
|
+
print({ kind: spec.kind, matched: out.matched, capped: out.capped, window: out.window, sample: out.sample }, flags);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* trodo traces export [filter flags] [--include-trace] [--format trodo|openai|dpo] [--target output|expected] [-o file]
|
|
26
|
+
* One JSON line per unit (the row a dataset would hold, with its evaluations),
|
|
27
|
+
* or one fine-tuning example per line. The summary goes to stderr.
|
|
28
|
+
*/
|
|
29
|
+
async function exportTraces(_args, flags) {
|
|
30
|
+
const api = client(config.resolve(flags));
|
|
31
|
+
const spec = await buildSpec(api, flags, flags.kind || 'runs');
|
|
32
|
+
const body = { ...spec };
|
|
33
|
+
if (flags.format) body.format = String(flags.format);
|
|
34
|
+
if (flags.target) body.target = String(flags.target);
|
|
35
|
+
const out = sink(flags.out && flags.out !== true ? String(flags.out) : null);
|
|
36
|
+
let summary = null;
|
|
37
|
+
for await (const line of api.lines('POST', '/traces/export', { body })) {
|
|
38
|
+
if (line && line._trodo) { summary = line._trodo; continue; }
|
|
39
|
+
// eslint-disable-next-line no-await-in-loop
|
|
40
|
+
await out.write(line);
|
|
41
|
+
}
|
|
42
|
+
await out.close();
|
|
43
|
+
if (summary) process.stderr.write(`${summary.rows} ${spec.kind}${summary.skipped ? `, ${summary.skipped} skipped (no input or no target)` : ''}${flags.out ? ` → ${flags.out}` : ''}\n`);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
module.exports = { count, export: exportTraces, buildSpec };
|
package/lib/config.js
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Where the CLI finds its key and server: TRODO_API_KEY / TRODO_API_URL, else
|
|
5
|
+
* ~/.trodo/config.json (written by `trodo login`, mode 600). A key is a Trodo
|
|
6
|
+
* API key (trodo_mk_…, Settings → MCP); it acts as the person who made
|
|
7
|
+
* it, on its team, with its scopes.
|
|
8
|
+
*/
|
|
9
|
+
const fs = require('fs');
|
|
10
|
+
const os = require('os');
|
|
11
|
+
const path = require('path');
|
|
12
|
+
|
|
13
|
+
const DEFAULT_URL = 'https://api.trodo.ai';
|
|
14
|
+
|
|
15
|
+
function file() {
|
|
16
|
+
return process.env.TRODO_CONFIG || path.join(os.homedir(), '.trodo', 'config.json');
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function read() {
|
|
20
|
+
try { return JSON.parse(fs.readFileSync(file(), 'utf8')); } catch { return {}; }
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function write(cfg) {
|
|
24
|
+
const f = file();
|
|
25
|
+
fs.mkdirSync(path.dirname(f), { recursive: true, mode: 0o700 });
|
|
26
|
+
fs.writeFileSync(f, `${JSON.stringify(cfg, null, 2)}\n`, { mode: 0o600 });
|
|
27
|
+
return f;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function resolve(flags = {}) {
|
|
31
|
+
const cfg = read();
|
|
32
|
+
const key = flags.key || process.env.TRODO_API_KEY || cfg.api_key || null;
|
|
33
|
+
const url = String(flags.url || process.env.TRODO_API_URL || cfg.api_url || DEFAULT_URL).replace(/\/+$/, '');
|
|
34
|
+
return { key, url };
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
module.exports = { read, write, resolve, file, DEFAULT_URL };
|
package/lib/filters.js
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Filter flags → the filter the Traces page and dataset sources use
|
|
5
|
+
* ({ modularFilters, time, eval_conditions }), so `trodo traces export`, a
|
|
6
|
+
* dataset source and the Traces page all mean the same thing.
|
|
7
|
+
*
|
|
8
|
+
* --since 7d | 24h | 30d | 3m | today | 90d (default 7d)
|
|
9
|
+
* --from ISO --to ISO
|
|
10
|
+
* --agent NAME (repeatable)
|
|
11
|
+
* --where "status=error" = != ~ (contains) > >= < <= e.g. total_cost>0.05,
|
|
12
|
+
* span.model~gpt-4o, metadata.tier=pro, attributes.region!=eu
|
|
13
|
+
* --eval "NAME:COND" COND: failed | passed | not-failed | marked-wrong |
|
|
14
|
+
* not-marked-wrong | marked-right | score<0.5 | reason=A,B |
|
|
15
|
+
* "STEP.FIELD=value" | "STEP.FIELD<3" NAME may be "any"
|
|
16
|
+
* --sample PCT, --limit N
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
const { list } = require('./args');
|
|
20
|
+
|
|
21
|
+
const OPS = [
|
|
22
|
+
['>=', 'greater_than_equal'], ['<=', 'less_than_equal'], ['!=', 'is_not'],
|
|
23
|
+
['>', 'greater_than'], ['<', 'less_than'], ['~', 'contains'], ['=', 'is'],
|
|
24
|
+
];
|
|
25
|
+
const NUM = { '<': 'lt', '<=': 'lte', '>': 'gt', '>=': 'gte', '=': 'eq' };
|
|
26
|
+
|
|
27
|
+
function fail(msg) { const e = new Error(msg); e.exitCode = 2; throw e; }
|
|
28
|
+
|
|
29
|
+
function timeOf(flags) {
|
|
30
|
+
if (flags.from || flags.to) {
|
|
31
|
+
const from = flags.from ? new Date(flags.from) : null;
|
|
32
|
+
const to = flags.to ? new Date(flags.to) : null;
|
|
33
|
+
if ((from && Number.isNaN(from.getTime())) || (to && Number.isNaN(to.getTime()))) fail('--from / --to must be dates (2026-10-01 or ISO)');
|
|
34
|
+
return { preset: 'custom', ...(from ? { from: from.toISOString() } : {}), ...(to ? { to: to.toISOString() } : {}) };
|
|
35
|
+
}
|
|
36
|
+
const since = String(flags.since || '7d');
|
|
37
|
+
if (!/^(\d+[mhd]|today|yesterday|\d+m)$/.test(since)) fail('--since takes 24h, 7d, 30d, 3m, 6m, 12m, today or yesterday');
|
|
38
|
+
return { preset: since };
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** "status=error" → a Traces filter chip. */
|
|
42
|
+
function whereChip(expr) {
|
|
43
|
+
const s = String(expr);
|
|
44
|
+
for (const [sym, operator] of OPS) {
|
|
45
|
+
const i = s.indexOf(sym);
|
|
46
|
+
if (i > 0) {
|
|
47
|
+
const field = s.slice(0, i).trim();
|
|
48
|
+
const raw = s.slice(i + sym.length).trim();
|
|
49
|
+
if (!field || raw === '') fail(`--where "${s}": write it as field${sym}value`);
|
|
50
|
+
const isNum = /^-?\d+(\.\d+)?$/.test(raw);
|
|
51
|
+
let value = raw;
|
|
52
|
+
if (operator === 'is' || operator === 'is_not') value = raw.split(',').map((x) => x.trim()).filter(Boolean);
|
|
53
|
+
else if (operator.startsWith('greater') || operator.startsWith('less')) {
|
|
54
|
+
if (!isNum) fail(`--where "${s}": ${sym} needs a number`);
|
|
55
|
+
value = Number(raw);
|
|
56
|
+
}
|
|
57
|
+
return { field, operator, value, category: 'global' };
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return fail(`--where "${s}": use =, !=, ~ (contains), >, >=, < or <=`);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function findEval(evals, name) {
|
|
64
|
+
if (/^any$/i.test(name)) return null;
|
|
65
|
+
const n = name.toLowerCase();
|
|
66
|
+
const exact = evals.filter((e) => e.id === name || e.name.toLowerCase() === n);
|
|
67
|
+
if (exact.length === 1) return exact[0];
|
|
68
|
+
const part = evals.filter((e) => e.name.toLowerCase().includes(n));
|
|
69
|
+
if (part.length === 1) return part[0];
|
|
70
|
+
if (!part.length) fail(`No eval called "${name}". \`trodo evals list\` shows them.`);
|
|
71
|
+
return fail(`"${name}" matches ${part.length} evals: ${part.map((e) => e.name).join(', ')}. Use more of the name.`);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** "Grounded:failed", "any:score<0.4", "Router:route.choice=billing" → an eval condition. */
|
|
75
|
+
function evalCondition(expr, evals) {
|
|
76
|
+
const s = String(expr);
|
|
77
|
+
const colon = s.indexOf(':');
|
|
78
|
+
if (colon <= 0) fail(`--eval "${s}": write it as NAME:CONDITION (e.g. "Grounded:failed")`);
|
|
79
|
+
const ev = findEval(evals, s.slice(0, colon).trim());
|
|
80
|
+
const base = ev ? { lineage_id: ev.id } : {};
|
|
81
|
+
const cond = s.slice(colon + 1).trim();
|
|
82
|
+
const words = {
|
|
83
|
+
failed: { verdict: 'fail' }, passed: { verdict: 'pass' }, 'not-failed': { verdict: 'fail', negate: true },
|
|
84
|
+
'marked-wrong': { marked: 'wrong' }, 'not-marked-wrong': { marked: 'wrong', negate: true }, 'marked-right': { marked: 'right' },
|
|
85
|
+
};
|
|
86
|
+
if (words[cond]) return { ...base, ...words[cond] };
|
|
87
|
+
let m = /^score\s*(<=|>=|<|>|=)\s*(-?\d+(\.\d+)?)$/.exec(cond);
|
|
88
|
+
if (m) return { ...base, score: { op: NUM[m[1]], value: Number(m[2]) } };
|
|
89
|
+
m = /^reason\s*=\s*(.+)$/.exec(cond);
|
|
90
|
+
if (m) return { ...base, reason_codes: m[1].split(',').map((x) => x.trim()).filter(Boolean) };
|
|
91
|
+
m = /^(.+?)\.([A-Za-z_][\w]*)\s*(<=|>=|<|>|!=|=)\s*(.+)$/.exec(cond);
|
|
92
|
+
if (m) {
|
|
93
|
+
if (!ev) fail(`--eval "${s}": a step needs a named eval, not "any"`);
|
|
94
|
+
const [, stepRef, field, op, rawVal] = m;
|
|
95
|
+
const steps = ev.steps || [];
|
|
96
|
+
const ref = stepRef.trim().toLowerCase();
|
|
97
|
+
const byStart = steps.filter((st) => st.name.toLowerCase().startsWith(ref));
|
|
98
|
+
const step = steps.find((st) => st.id === stepRef || st.name.toLowerCase() === ref) || (byStart.length === 1 ? byStart[0] : null);
|
|
99
|
+
if (!step) fail(`--eval "${s}": "${ev.name}" has no step "${stepRef}". Its steps: ${(ev.steps || []).map((st) => st.name).join(', ') || 'none that answer'}`);
|
|
100
|
+
const f = (step.fields || []).find((x) => x.name === field);
|
|
101
|
+
if (!f) fail(`--eval "${s}": step "${step.name}" answers ${(step.fields || []).map((x) => x.name).join(', ')}, not ${field}`);
|
|
102
|
+
const val = rawVal.trim();
|
|
103
|
+
if (f.type === 'boolean') {
|
|
104
|
+
const yes = /^(true|yes|1)$/i.test(val); const no = /^(false|no|0)$/i.test(val);
|
|
105
|
+
if (!yes && !no) fail(`--eval "${s}": ${field} is yes/no`);
|
|
106
|
+
return { ...base, step: { node_id: step.id, field, op: op === '!=' ? 'neq' : 'eq', value: yes } };
|
|
107
|
+
}
|
|
108
|
+
if (f.type === 'number') {
|
|
109
|
+
if (!/^-?\d+(\.\d+)?$/.test(val)) fail(`--eval "${s}": ${field} is a number`);
|
|
110
|
+
return { ...base, step: { node_id: step.id, field, op: op === '!=' ? 'neq' : NUM[op], value: Number(val) } };
|
|
111
|
+
}
|
|
112
|
+
if (f.type === 'enum') {
|
|
113
|
+
return { ...base, step: { node_id: step.id, field, op: op === '!=' ? 'not_in' : 'in', value: val.split(',').map((x) => x.trim()) } };
|
|
114
|
+
}
|
|
115
|
+
return { ...base, step: { node_id: step.id, field, op: op === '!=' ? 'neq' : 'eq', value: val } };
|
|
116
|
+
}
|
|
117
|
+
return fail(`--eval "${s}": unknown condition "${cond}". Use failed, passed, not-failed, marked-wrong, not-marked-wrong, marked-right, score<0.5, reason=CODE or STEP.FIELD=value`);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* The flags → { kind, filter, options }. `evals` is GET /evals's list,
|
|
122
|
+
* needed only when --eval names one.
|
|
123
|
+
*/
|
|
124
|
+
function specFromFlags(flags, { kind = flags.kind || 'runs', evals = [] } = {}) {
|
|
125
|
+
if (!['spans', 'runs', 'conversations', 'verdicts'].includes(kind)) fail('--kind is spans, runs, conversations or verdicts');
|
|
126
|
+
const time = timeOf(flags);
|
|
127
|
+
const agents = list(flags.agent);
|
|
128
|
+
const where = list(flags.where).map(whereChip);
|
|
129
|
+
const conds = list(flags.eval).map((e) => evalCondition(e, evals));
|
|
130
|
+
const options = {};
|
|
131
|
+
if (flags.limit !== undefined) {
|
|
132
|
+
const n = parseInt(flags.limit, 10);
|
|
133
|
+
if (!(n > 0)) fail('--limit must be a positive number');
|
|
134
|
+
options.limit = n;
|
|
135
|
+
}
|
|
136
|
+
if (flags.sample !== undefined) options.sample_pct = Number(flags.sample);
|
|
137
|
+
if (flags['include-trace']) options.include_trace = true;
|
|
138
|
+
|
|
139
|
+
if (kind === 'verdicts') {
|
|
140
|
+
const outcomes = list(flags.outcome).flatMap((o) => o.split(',')).map((o) => o.trim().replace(/-/g, '_')).filter(Boolean);
|
|
141
|
+
const ids = conds.map((c) => c.lineage_id).filter(Boolean);
|
|
142
|
+
const filter = { lineage_ids: ids, outcomes: outcomes.length ? outcomes : ['failed'], time };
|
|
143
|
+
if (flags.step) options.node_id = String(flags.step);
|
|
144
|
+
return { kind, filter, options };
|
|
145
|
+
}
|
|
146
|
+
const modularFilters = [];
|
|
147
|
+
if (agents.length) modularFilters.push({ field: 'agent_name', operator: 'is', value: agents, category: kind === 'spans' ? 'global' : 'agents' });
|
|
148
|
+
modularFilters.push(...where);
|
|
149
|
+
const filter = { modularFilters, time };
|
|
150
|
+
if (conds.length) filter.eval_conditions = conds;
|
|
151
|
+
if (kind === 'spans' && flags['any-span']) filter.span_kind = 'any';
|
|
152
|
+
return { kind, filter, options };
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
const needsEvals = (flags) => list(flags.eval).some((e) => !/^any:/i.test(e));
|
|
156
|
+
|
|
157
|
+
module.exports = { specFromFlags, whereChip, evalCondition, timeOf, needsEvals };
|
package/lib/http.js
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/** The API, over fetch (Node 18+). Every call goes to {url}/api/v1. */
|
|
4
|
+
|
|
5
|
+
class ApiError extends Error {
|
|
6
|
+
constructor(status, body) {
|
|
7
|
+
const msg = (body && (body.message || body.error)) || `HTTP ${status}`;
|
|
8
|
+
super(typeof msg === 'string' ? msg : JSON.stringify(msg));
|
|
9
|
+
this.status = status;
|
|
10
|
+
this.body = body;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function client({ url, key, fetchImpl = globalThis.fetch }) {
|
|
15
|
+
if (!key) {
|
|
16
|
+
const e = new Error('No API key. Run `trodo login`, or set TRODO_API_KEY (a trodo_mk_ key from Settings → MCP).');
|
|
17
|
+
e.exitCode = 2;
|
|
18
|
+
throw e;
|
|
19
|
+
}
|
|
20
|
+
const base = `${url}/api/v1`;
|
|
21
|
+
const headers = { Authorization: `Bearer ${key}`, 'User-Agent': 'trodo-cli/0.2', Accept: 'application/json' };
|
|
22
|
+
|
|
23
|
+
async function request(method, p, { query, body } = {}) {
|
|
24
|
+
const qs = query ? `?${new URLSearchParams(Object.entries(query).filter(([, v]) => v !== undefined && v !== null && v !== '').map(([k, v]) => [k, String(v)]))}` : '';
|
|
25
|
+
const res = await fetchImpl(`${base}${p}${qs}`, {
|
|
26
|
+
method,
|
|
27
|
+
headers: body !== undefined ? { ...headers, 'Content-Type': 'application/json' } : headers,
|
|
28
|
+
body: body !== undefined ? JSON.stringify(body) : undefined,
|
|
29
|
+
});
|
|
30
|
+
return res;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
async function json(method, p, opts) {
|
|
34
|
+
const res = await request(method, p, opts);
|
|
35
|
+
const text = await res.text();
|
|
36
|
+
let data = null;
|
|
37
|
+
try { data = text ? JSON.parse(text) : null; } catch { data = { message: text.slice(0, 300) }; }
|
|
38
|
+
if (!res.ok) throw new ApiError(res.status, data);
|
|
39
|
+
return data;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** NDJSON lines from a response, one parsed object at a time. */
|
|
43
|
+
async function* lines(method, p, opts) {
|
|
44
|
+
const res = await request(method, p, opts);
|
|
45
|
+
if (!res.ok) {
|
|
46
|
+
const text = await res.text();
|
|
47
|
+
let data; try { data = JSON.parse(text); } catch { data = { message: text.slice(0, 300) }; }
|
|
48
|
+
throw new ApiError(res.status, data);
|
|
49
|
+
}
|
|
50
|
+
const decoder = new TextDecoder();
|
|
51
|
+
let buf = '';
|
|
52
|
+
for await (const chunk of res.body) {
|
|
53
|
+
buf += decoder.decode(chunk, { stream: true });
|
|
54
|
+
let i;
|
|
55
|
+
while ((i = buf.indexOf('\n')) >= 0) {
|
|
56
|
+
const line = buf.slice(0, i).trim();
|
|
57
|
+
buf = buf.slice(i + 1);
|
|
58
|
+
if (line) yield JSON.parse(line);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
if (buf.trim()) yield JSON.parse(buf.trim());
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
return {
|
|
65
|
+
get: (p, query) => json('GET', p, { query }),
|
|
66
|
+
post: (p, body, query) => json('POST', p, { body, query }),
|
|
67
|
+
patch: (p, body) => json('PATCH', p, { body }),
|
|
68
|
+
lines,
|
|
69
|
+
raw: request,
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
module.exports = { client, ApiError };
|
package/lib/main.js
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* trodo — Trodo from a terminal, a script, CI or a coding agent.
|
|
5
|
+
* Every command prints JSON (pretty on a terminal) unless --table is given.
|
|
6
|
+
*/
|
|
7
|
+
const { parse } = require('./args');
|
|
8
|
+
const { ApiError } = require('./http');
|
|
9
|
+
|
|
10
|
+
const pkg = require('../package.json');
|
|
11
|
+
|
|
12
|
+
const HELP = `trodo ${pkg.version} — traces, datasets and agents from the terminal
|
|
13
|
+
|
|
14
|
+
trodo login [--url https://api.trodo.ai] [--no-browser] sign in through the app
|
|
15
|
+
trodo login --key trodo_mk_… or with a key (CI)
|
|
16
|
+
trodo whoami | logout
|
|
17
|
+
|
|
18
|
+
trodo evals list eval names and their steps, for --eval
|
|
19
|
+
trodo traces count [filter]
|
|
20
|
+
trodo traces export [filter] [--include-trace] [--format trodo|openai|dpo] [--target output|expected] [-o file]
|
|
21
|
+
trodo runs list [filter] [--limit 50]
|
|
22
|
+
trodo runs get RUN_ID
|
|
23
|
+
|
|
24
|
+
trodo datasets list | get DATASET | versions DATASET
|
|
25
|
+
trodo datasets create NAME [--kind runs|spans|conversations|verdicts [filter] [--live] [--include-trace]]
|
|
26
|
+
trodo datasets add DATASET --kind KIND [filter] [--live] [--include-trace]
|
|
27
|
+
trodo datasets append DATASET [-f rows.jsonl] rows, or {run_id}/{span_id}/{conversation_id} lines
|
|
28
|
+
trodo datasets update DATASET [-f updates.jsonl] {id, expected?, output?, metadata?} lines, one version
|
|
29
|
+
trodo datasets export DATASET [--rows passed|failed|expected|no-expected|marked-wrong|all]
|
|
30
|
+
[--eval NAME] [--min-score N] [--format trodo|openai|dpo] [--target ...] [--version N] [-o file]
|
|
31
|
+
|
|
32
|
+
trodo agents list | executions AGENT | execution ID
|
|
33
|
+
trodo agents run AGENT [--input JSON] [--wait]
|
|
34
|
+
|
|
35
|
+
Filter (traces, runs, datasets create/add):
|
|
36
|
+
--kind spans|runs|conversations|verdicts (default runs)
|
|
37
|
+
--since 24h|7d|30d|3m|today or --from DATE --to DATE (default 7d)
|
|
38
|
+
--agent NAME repeatable
|
|
39
|
+
--where "status=error" = != ~ > >= < <= on any Traces property:
|
|
40
|
+
total_cost>0.05 span.model~gpt-4o metadata.tier=pro
|
|
41
|
+
--eval "NAME:COND" failed | passed | not-failed | marked-wrong | not-marked-wrong |
|
|
42
|
+
marked-right | score<0.5 | reason=CODE | STEP.FIELD=value | STEP.FIELD<3
|
|
43
|
+
NAME may be "any". Conditions AND together.
|
|
44
|
+
--outcome failed,passed,marked-wrong (verdicts) --step STEP_ID (verdicts, one step per row)
|
|
45
|
+
--limit N --sample PCT
|
|
46
|
+
|
|
47
|
+
Training data:
|
|
48
|
+
trodo traces export --kind spans --eval "any:passed" --eval "any:not-failed" --format openai --target output -o sft.jsonl
|
|
49
|
+
trodo datasets create "Good refunds" --kind runs --eval "Refund policy:passed" --include-trace
|
|
50
|
+
trodo datasets export "Good refunds" --rows passed --format openai --target output -o train.jsonl
|
|
51
|
+
firectl dataset create good-refunds train.jsonl
|
|
52
|
+
|
|
53
|
+
Auth: TRODO_API_KEY / TRODO_API_URL, or \`trodo login\` (~/.trodo/config.json).
|
|
54
|
+
A key is a Trodo API key (Settings → MCP); it acts as the person who made it, on its team, with its scopes.
|
|
55
|
+
Output: JSON by default; --table for a table. Exit codes: 0 ok, 1 API error, 2 usage.
|
|
56
|
+
`;
|
|
57
|
+
|
|
58
|
+
const COMMANDS = {
|
|
59
|
+
login: ['auth', 'login'], whoami: ['auth', 'whoami'], logout: ['auth', 'logout'],
|
|
60
|
+
};
|
|
61
|
+
const GROUPS = {
|
|
62
|
+
evals: ['list'],
|
|
63
|
+
traces: ['count', 'export'],
|
|
64
|
+
runs: ['list', 'get'],
|
|
65
|
+
datasets: ['list', 'get', 'create', 'add', 'append', 'update', 'export', 'rows', 'versions'],
|
|
66
|
+
agents: ['list', 'run', 'executions', 'execution'],
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
async function main(argv) {
|
|
70
|
+
const { flags, positional } = parse(argv);
|
|
71
|
+
const [cmd, sub, ...rest] = positional;
|
|
72
|
+
// `trodo --version` alone is the CLI's version; `datasets export X --version 3` is a dataset's.
|
|
73
|
+
if (flags.version === true && !cmd) { process.stdout.write(`${pkg.version}\n`); return 0; }
|
|
74
|
+
if (!cmd || flags.help || cmd === 'help') { process.stdout.write(HELP); return 0; }
|
|
75
|
+
try {
|
|
76
|
+
if (COMMANDS[cmd]) {
|
|
77
|
+
const [mod, fn] = COMMANDS[cmd];
|
|
78
|
+
// eslint-disable-next-line global-require, import/no-dynamic-require
|
|
79
|
+
await require(`./commands/${mod}`)[fn]([sub, ...rest].filter(Boolean), flags);
|
|
80
|
+
return process.exitCode || 0;
|
|
81
|
+
}
|
|
82
|
+
if (!GROUPS[cmd]) throw Object.assign(new Error(`Unknown command "${cmd}". \`trodo help\` lists them.`), { exitCode: 2 });
|
|
83
|
+
const action = sub || 'list';
|
|
84
|
+
if (!GROUPS[cmd].includes(action)) throw Object.assign(new Error(`Unknown: trodo ${cmd} ${action}. Try: ${GROUPS[cmd].map((a) => `trodo ${cmd} ${a}`).join(', ')}`), { exitCode: 2 });
|
|
85
|
+
// eslint-disable-next-line global-require, import/no-dynamic-require
|
|
86
|
+
await require(`./commands/${cmd}`)[action](rest, flags);
|
|
87
|
+
return process.exitCode || 0;
|
|
88
|
+
} catch (e) {
|
|
89
|
+
const msg = e instanceof ApiError ? `${e.message}${e.status ? ` (HTTP ${e.status})` : ''}` : e.message;
|
|
90
|
+
process.stderr.write(`trodo: ${msg}\n`);
|
|
91
|
+
if (flags.json) process.stdout.write(`${JSON.stringify({ error: msg, status: e.status || null, detail: e.body || null })}\n`);
|
|
92
|
+
return e.exitCode || 1;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
module.exports = { main, HELP };
|
package/lib/output.js
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* JSON by default (pretty on a terminal, one line when piped) so a coding
|
|
5
|
+
* agent can read every command; --table for a person.
|
|
6
|
+
*/
|
|
7
|
+
const fs = require('fs');
|
|
8
|
+
|
|
9
|
+
function print(data, flags = {}, columns = null) {
|
|
10
|
+
if (flags.table && columns && Array.isArray(data)) return process.stdout.write(table(data, columns));
|
|
11
|
+
const pretty = process.stdout.isTTY && !flags.json;
|
|
12
|
+
return process.stdout.write(`${JSON.stringify(data, null, pretty ? 2 : 0)}\n`);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function table(rows, columns) {
|
|
16
|
+
const cell = (v) => (v == null ? '' : typeof v === 'object' ? JSON.stringify(v) : String(v)).replace(/\s+/g, ' ');
|
|
17
|
+
const cut = (s, w) => (s.length > w ? `${s.slice(0, w - 1)}…` : s);
|
|
18
|
+
const widths = columns.map(([key, label, max = 40]) => Math.min(max, Math.max(label.length, ...rows.map((r) => cell(r[key]).length))));
|
|
19
|
+
const line = (vals) => vals.map((v, i) => cut(v, widths[i]).padEnd(widths[i])).join(' ').trimEnd();
|
|
20
|
+
return `${[line(columns.map(([, label]) => label)), ...rows.map((r) => line(columns.map(([key]) => cell(r[key]))))].join('\n')}\n`;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** Write lines to --out or stdout; returns { write(obj), close() }. */
|
|
24
|
+
function sink(out) {
|
|
25
|
+
const stream = out ? fs.createWriteStream(out) : process.stdout;
|
|
26
|
+
return {
|
|
27
|
+
write: (obj) => new Promise((resolve) => { if (stream.write(`${JSON.stringify(obj)}\n`)) resolve(); else stream.once('drain', resolve); }),
|
|
28
|
+
close: () => new Promise((resolve) => { if (out) stream.end(resolve); else resolve(); }),
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Read JSON lines (or one JSON array) from a file or stdin. */
|
|
33
|
+
async function readRows(file) {
|
|
34
|
+
let text;
|
|
35
|
+
if (file && file !== '-') text = fs.readFileSync(file, 'utf8');
|
|
36
|
+
else {
|
|
37
|
+
if (process.stdin.isTTY) { const e = new Error('Give rows with -f rows.jsonl, or pipe them in.'); e.exitCode = 2; throw e; }
|
|
38
|
+
const chunks = [];
|
|
39
|
+
for await (const c of process.stdin) chunks.push(c);
|
|
40
|
+
text = Buffer.concat(chunks).toString('utf8');
|
|
41
|
+
}
|
|
42
|
+
const t = text.trim();
|
|
43
|
+
if (!t) return [];
|
|
44
|
+
if (t.startsWith('[')) return JSON.parse(t);
|
|
45
|
+
return t.split('\n').map((l) => l.trim()).filter(Boolean).map((l, i) => {
|
|
46
|
+
try { return JSON.parse(l); } catch (e) { const err = new Error(`line ${i + 1} is not JSON: ${e.message}`); err.exitCode = 2; throw err; }
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
module.exports = { print, table, sink, readRows };
|
package/package.json
CHANGED
|
@@ -1,6 +1,30 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "trodo-cli",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.2.0",
|
|
4
|
+
"description": "Trodo from the terminal: pull traces by their eval results, build datasets, run labeling agents, export fine-tuning data",
|
|
5
|
+
"bin": {
|
|
6
|
+
"trodo": "bin/trodo.js"
|
|
7
|
+
},
|
|
8
|
+
"main": "./lib/main.js",
|
|
9
|
+
"files": [
|
|
10
|
+
"bin/",
|
|
11
|
+
"lib/",
|
|
12
|
+
"README.md"
|
|
13
|
+
],
|
|
14
|
+
"scripts": {
|
|
15
|
+
"test": "node --test test/"
|
|
16
|
+
},
|
|
17
|
+
"engines": {
|
|
18
|
+
"node": ">=18"
|
|
19
|
+
},
|
|
20
|
+
"keywords": [
|
|
21
|
+
"trodo",
|
|
22
|
+
"traces",
|
|
23
|
+
"datasets",
|
|
24
|
+
"fine-tuning",
|
|
25
|
+
"evals",
|
|
26
|
+
"cli",
|
|
27
|
+
"agents"
|
|
28
|
+
],
|
|
29
|
+
"license": "ISC"
|
|
30
|
+
}
|