trodo-cli 0.0.0-stage → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -2
- package/bin/trodo.js +7 -0
- package/lib/args.js +45 -0
- package/lib/commands/agents.js +63 -0
- package/lib/commands/api.js +37 -0
- package/lib/commands/auth.js +165 -0
- package/lib/commands/conversations.js +49 -0
- package/lib/commands/datasets.js +182 -0
- package/lib/commands/evals.js +138 -0
- package/lib/commands/issues.js +40 -0
- package/lib/commands/prompts.js +125 -0
- package/lib/commands/runs.js +118 -0
- package/lib/commands/setup.js +34 -0
- package/lib/commands/spans.js +38 -0
- package/lib/commands/traces.js +112 -0
- package/lib/commands/users.js +14 -0
- package/lib/common.js +69 -0
- package/lib/config.js +78 -0
- package/lib/filters.js +157 -0
- package/lib/http.js +74 -0
- package/lib/main.js +137 -0
- package/lib/output.js +61 -0
- package/package.json +29 -4
- package/skills/trodo/SKILL.md +121 -0
- package/skills/trodo-datasets/SKILL.md +116 -0
package/lib/common.js
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/** Small pieces every command uses. */
|
|
4
|
+
|
|
5
|
+
const usage = (msg) => { const e = new Error(msg); e.exitCode = 2; return e; };
|
|
6
|
+
|
|
7
|
+
/** The one named `ref` (an id, an exact name, or a name in any case) out of `all`. */
|
|
8
|
+
function pick(all, ref, { kind, listCmd, nameKey = 'name', idKeys = ['id'] } = {}) {
|
|
9
|
+
if (ref == null || ref === '' || ref === true) throw usage(`Name the ${kind} (its name or id).`);
|
|
10
|
+
const r = String(ref);
|
|
11
|
+
const hit = all.find((x) => idKeys.some((k) => x[k] != null && String(x[k]) === r))
|
|
12
|
+
|| all.find((x) => x[nameKey] === r)
|
|
13
|
+
|| all.find((x) => String(x[nameKey] || '').toLowerCase() === r.toLowerCase());
|
|
14
|
+
if (!hit) throw usage(`No ${kind} "${r}". \`${listCmd}\` shows them.`);
|
|
15
|
+
return hit;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* A filter spec (filters.specFromFlags) as the query the Runs, Spans and
|
|
20
|
+
* Conversations lists read: the time window plus the Traces filter chips,
|
|
21
|
+
* eval conditions included, sent the way the app sends them.
|
|
22
|
+
*/
|
|
23
|
+
function listQuery(spec, windowOf) {
|
|
24
|
+
const mf = [...(spec.filter.modularFilters || [])];
|
|
25
|
+
if (spec.filter.eval_conditions) mf.push({ category: 'evals', field: 'eval_conditions', operator: 'is', value: spec.filter.eval_conditions });
|
|
26
|
+
const q = { ...windowOf(spec.filter.time) };
|
|
27
|
+
// The lists decode this once more (decodeURIComponent then JSON), as the app sends it.
|
|
28
|
+
if (mf.length) q.modularFilters = encodeURIComponent(JSON.stringify(mf));
|
|
29
|
+
return q;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const int = (v, dflt, max) => {
|
|
33
|
+
const n = Number.parseInt(v, 10);
|
|
34
|
+
if (!Number.isFinite(n) || n < 1) return dflt;
|
|
35
|
+
return max ? Math.min(n, max) : n;
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* One page, or with --all every page (`max` at a time, up to --max-items,
|
|
40
|
+
* 10,000 by default): `page(limit, offset)` returns that page's rows.
|
|
41
|
+
*/
|
|
42
|
+
async function pages(flags, page, { dflt = 50, max = 200 } = {}) {
|
|
43
|
+
if (!flags.all) return page(int(flags.limit, dflt, max), flags.offset);
|
|
44
|
+
const cap = int(flags['max-items'], 10000, 1000000);
|
|
45
|
+
const out = [];
|
|
46
|
+
let offset = Number.parseInt(flags.offset, 10) || 0;
|
|
47
|
+
for (;;) {
|
|
48
|
+
// eslint-disable-next-line no-await-in-loop
|
|
49
|
+
const rows = await page(max, offset);
|
|
50
|
+
out.push(...rows);
|
|
51
|
+
if (rows.length < max || out.length >= cap) break;
|
|
52
|
+
offset += rows.length;
|
|
53
|
+
}
|
|
54
|
+
return out.slice(0, cap);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Every eval's verdicts on one unit (GET /eval3/units/:target/:id/verdicts,
|
|
59
|
+
* grouped by eval) as one list, newest first, each carrying its eval's name.
|
|
60
|
+
*/
|
|
61
|
+
function unitVerdicts(out) {
|
|
62
|
+
if (!out) return undefined;
|
|
63
|
+
if (Array.isArray(out.verdicts)) return out.verdicts;
|
|
64
|
+
const rows = [];
|
|
65
|
+
for (const l of out.lineages || []) for (const v of l.verdicts || []) rows.push({ lineage_name: l.name, ...v });
|
|
66
|
+
return rows.sort((a, b) => String(b.created_at || '').localeCompare(String(a.created_at || '')));
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
module.exports = { usage, pick, listQuery, int, pages, unitVerdicts };
|
package/lib/config.js
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Where the CLI finds its key and server: TRODO_API_KEY / TRODO_API_URL, else
|
|
5
|
+
* ~/.trodo/config.json (written by `trodo login`, mode 600). A key is a Trodo
|
|
6
|
+
* API key (trodo_mk_…, Settings → MCP); it acts as the person who made it, on
|
|
7
|
+
* its team, with its scopes.
|
|
8
|
+
*
|
|
9
|
+
* A key belongs to one team, so the file keeps one per team signed in to:
|
|
10
|
+
* { "current_team": "12", "teams": { "12": { "name", "api_key", "api_url" }, … } }
|
|
11
|
+
* `trodo switch TEAM` changes the current one; `--team TEAM` picks one for a
|
|
12
|
+
* single command. A file from 0.2 ({ api_key, api_url }) still works.
|
|
13
|
+
*/
|
|
14
|
+
const fs = require('fs');
|
|
15
|
+
const os = require('os');
|
|
16
|
+
const path = require('path');
|
|
17
|
+
|
|
18
|
+
const DEFAULT_URL = 'https://api.trodo.ai';
|
|
19
|
+
|
|
20
|
+
function file() {
|
|
21
|
+
return process.env.TRODO_CONFIG || path.join(os.homedir(), '.trodo', 'config.json');
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function read() {
|
|
25
|
+
try { return JSON.parse(fs.readFileSync(file(), 'utf8')); } catch { return {}; }
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function write(cfg) {
|
|
29
|
+
const f = file();
|
|
30
|
+
fs.mkdirSync(path.dirname(f), { recursive: true, mode: 0o700 });
|
|
31
|
+
fs.writeFileSync(f, `${JSON.stringify(cfg, null, 2)}\n`, { mode: 0o600 });
|
|
32
|
+
// `mode` applies only when the file is created: an older file keeps whatever it had.
|
|
33
|
+
try { fs.chmodSync(f, 0o600); } catch { /* not ours to change (a shared mount); the key is still written */ }
|
|
34
|
+
return f;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** The saved teams as a list, the current one marked. */
|
|
38
|
+
function teams(cfg = read()) {
|
|
39
|
+
return Object.entries(cfg.teams || {}).map(([id, t]) => ({ id, name: t.name || id, api_url: t.api_url || cfg.api_url || DEFAULT_URL, current: String(cfg.current_team) === id, api_key: t.api_key }));
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** A saved team by id or name (exact, then any case); null when none. */
|
|
43
|
+
function findTeam(cfg, ref) {
|
|
44
|
+
const all = teams(cfg);
|
|
45
|
+
const r = String(ref);
|
|
46
|
+
return all.find((t) => t.id === r) || all.find((t) => t.name === r) || all.find((t) => t.name.toLowerCase() === r.toLowerCase()) || null;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function resolve(flags = {}) {
|
|
50
|
+
const cfg = read();
|
|
51
|
+
const envUrl = flags.url || process.env.TRODO_API_URL;
|
|
52
|
+
const finish = (key, url) => ({ key, url: String(envUrl || url || cfg.api_url || DEFAULT_URL).replace(/\/+$/, '') });
|
|
53
|
+
if (flags.key && flags.key !== true) return finish(String(flags.key));
|
|
54
|
+
if (process.env.TRODO_API_KEY) return finish(process.env.TRODO_API_KEY);
|
|
55
|
+
if (flags.team && flags.team !== true) {
|
|
56
|
+
const t = findTeam(cfg, flags.team);
|
|
57
|
+
if (!t) {
|
|
58
|
+
const e = new Error(`Not signed in to a team called "${flags.team}". \`trodo teams\` lists yours; \`trodo login\` adds one.`);
|
|
59
|
+
e.exitCode = 2;
|
|
60
|
+
throw e;
|
|
61
|
+
}
|
|
62
|
+
return finish(t.api_key, t.api_url);
|
|
63
|
+
}
|
|
64
|
+
const cur = cfg.teams && cfg.current_team != null ? cfg.teams[String(cfg.current_team)] : null;
|
|
65
|
+
if (cur) return finish(cur.api_key, cur.api_url);
|
|
66
|
+
return finish(cfg.api_key || null);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Save a team's key and make it the current team. */
|
|
70
|
+
function saveTeam({ id, name, key, url }) {
|
|
71
|
+
const cfg = read();
|
|
72
|
+
const t = { ...(cfg.teams || {}) };
|
|
73
|
+
t[String(id)] = { name, api_key: key, api_url: url };
|
|
74
|
+
const next = { api_url: url, current_team: String(id), teams: t };
|
|
75
|
+
return write(next);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
module.exports = { read, write, resolve, file, teams, findTeam, saveTeam, DEFAULT_URL };
|
package/lib/filters.js
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Filter flags → the filter the Traces page and dataset sources use
|
|
5
|
+
* ({ modularFilters, time, eval_conditions }), so `trodo traces export`, a
|
|
6
|
+
* dataset source and the Traces page all mean the same thing.
|
|
7
|
+
*
|
|
8
|
+
* --since 7d | 24h | 30d | 3m | today | 90d (default 7d)
|
|
9
|
+
* --from ISO --to ISO
|
|
10
|
+
* --agent NAME (repeatable)
|
|
11
|
+
* --where "status=error" = != ~ (contains) > >= < <= e.g. total_cost>0.05,
|
|
12
|
+
* span.model~gpt-4o, metadata.tier=pro, attributes.region!=eu
|
|
13
|
+
* --eval "NAME:COND" COND: failed | passed | not-failed | marked-wrong |
|
|
14
|
+
* not-marked-wrong | marked-right | score<0.5 | reason=A,B |
|
|
15
|
+
* "STEP.FIELD=value" | "STEP.FIELD<3" NAME may be "any"
|
|
16
|
+
* --sample PCT, --limit N
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
const { list } = require('./args');
|
|
20
|
+
|
|
21
|
+
const OPS = [
|
|
22
|
+
['>=', 'greater_than_equal'], ['<=', 'less_than_equal'], ['!=', 'is_not'],
|
|
23
|
+
['>', 'greater_than'], ['<', 'less_than'], ['~', 'contains'], ['=', 'is'],
|
|
24
|
+
];
|
|
25
|
+
const NUM = { '<': 'lt', '<=': 'lte', '>': 'gt', '>=': 'gte', '=': 'eq' };
|
|
26
|
+
|
|
27
|
+
function fail(msg) { const e = new Error(msg); e.exitCode = 2; throw e; }
|
|
28
|
+
|
|
29
|
+
function timeOf(flags) {
|
|
30
|
+
if (flags.from || flags.to) {
|
|
31
|
+
const from = flags.from ? new Date(flags.from) : null;
|
|
32
|
+
const to = flags.to ? new Date(flags.to) : null;
|
|
33
|
+
if ((from && Number.isNaN(from.getTime())) || (to && Number.isNaN(to.getTime()))) fail('--from / --to must be dates (2026-10-01 or ISO)');
|
|
34
|
+
return { preset: 'custom', ...(from ? { from: from.toISOString() } : {}), ...(to ? { to: to.toISOString() } : {}) };
|
|
35
|
+
}
|
|
36
|
+
const since = String(flags.since || '7d');
|
|
37
|
+
if (!/^(\d+[mhd]|today|yesterday|\d+m)$/.test(since)) fail('--since takes 24h, 7d, 30d, 3m, 6m, 12m, today or yesterday');
|
|
38
|
+
return { preset: since };
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** "status=error" → a Traces filter chip. */
|
|
42
|
+
function whereChip(expr) {
|
|
43
|
+
const s = String(expr);
|
|
44
|
+
for (const [sym, operator] of OPS) {
|
|
45
|
+
const i = s.indexOf(sym);
|
|
46
|
+
if (i > 0) {
|
|
47
|
+
const field = s.slice(0, i).trim();
|
|
48
|
+
const raw = s.slice(i + sym.length).trim();
|
|
49
|
+
if (!field || raw === '') fail(`--where "${s}": write it as field${sym}value`);
|
|
50
|
+
const isNum = /^-?\d+(\.\d+)?$/.test(raw);
|
|
51
|
+
let value = raw;
|
|
52
|
+
if (operator === 'is' || operator === 'is_not') value = raw.split(',').map((x) => x.trim()).filter(Boolean);
|
|
53
|
+
else if (operator.startsWith('greater') || operator.startsWith('less')) {
|
|
54
|
+
if (!isNum) fail(`--where "${s}": ${sym} needs a number`);
|
|
55
|
+
value = Number(raw);
|
|
56
|
+
}
|
|
57
|
+
return { field, operator, value, category: 'global' };
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return fail(`--where "${s}": use =, !=, ~ (contains), >, >=, < or <=`);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function findEval(evals, name) {
|
|
64
|
+
if (/^any$/i.test(name)) return null;
|
|
65
|
+
const n = name.toLowerCase();
|
|
66
|
+
const exact = evals.filter((e) => e.id === name || e.name.toLowerCase() === n);
|
|
67
|
+
if (exact.length === 1) return exact[0];
|
|
68
|
+
const part = evals.filter((e) => e.name.toLowerCase().includes(n));
|
|
69
|
+
if (part.length === 1) return part[0];
|
|
70
|
+
if (!part.length) fail(`No eval called "${name}". \`trodo evals list\` shows them.`);
|
|
71
|
+
return fail(`"${name}" matches ${part.length} evals: ${part.map((e) => e.name).join(', ')}. Use more of the name.`);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** "Grounded:failed", "any:score<0.4", "Router:route.choice=billing" → an eval condition. */
|
|
75
|
+
function evalCondition(expr, evals) {
|
|
76
|
+
const s = String(expr);
|
|
77
|
+
const colon = s.indexOf(':');
|
|
78
|
+
if (colon <= 0) fail(`--eval "${s}": write it as NAME:CONDITION (e.g. "Grounded:failed")`);
|
|
79
|
+
const ev = findEval(evals, s.slice(0, colon).trim());
|
|
80
|
+
const base = ev ? { lineage_id: ev.id } : {};
|
|
81
|
+
const cond = s.slice(colon + 1).trim();
|
|
82
|
+
const words = {
|
|
83
|
+
failed: { verdict: 'fail' }, passed: { verdict: 'pass' }, 'not-failed': { verdict: 'fail', negate: true },
|
|
84
|
+
'marked-wrong': { marked: 'wrong' }, 'not-marked-wrong': { marked: 'wrong', negate: true }, 'marked-right': { marked: 'right' },
|
|
85
|
+
};
|
|
86
|
+
if (words[cond]) return { ...base, ...words[cond] };
|
|
87
|
+
let m = /^score\s*(<=|>=|<|>|=)\s*(-?\d+(\.\d+)?)$/.exec(cond);
|
|
88
|
+
if (m) return { ...base, score: { op: NUM[m[1]], value: Number(m[2]) } };
|
|
89
|
+
m = /^reason\s*=\s*(.+)$/.exec(cond);
|
|
90
|
+
if (m) return { ...base, reason_codes: m[1].split(',').map((x) => x.trim()).filter(Boolean) };
|
|
91
|
+
m = /^(.+?)\.([A-Za-z_][\w]*)\s*(<=|>=|<|>|!=|=)\s*(.+)$/.exec(cond);
|
|
92
|
+
if (m) {
|
|
93
|
+
if (!ev) fail(`--eval "${s}": a step needs a named eval, not "any"`);
|
|
94
|
+
const [, stepRef, field, op, rawVal] = m;
|
|
95
|
+
const steps = ev.steps || [];
|
|
96
|
+
const ref = stepRef.trim().toLowerCase();
|
|
97
|
+
const byStart = steps.filter((st) => st.name.toLowerCase().startsWith(ref));
|
|
98
|
+
const step = steps.find((st) => st.id === stepRef || st.name.toLowerCase() === ref) || (byStart.length === 1 ? byStart[0] : null);
|
|
99
|
+
if (!step) fail(`--eval "${s}": "${ev.name}" has no step "${stepRef}". Its steps: ${(ev.steps || []).map((st) => st.name).join(', ') || 'none that answer'}`);
|
|
100
|
+
const f = (step.fields || []).find((x) => x.name === field);
|
|
101
|
+
if (!f) fail(`--eval "${s}": step "${step.name}" answers ${(step.fields || []).map((x) => x.name).join(', ')}, not ${field}`);
|
|
102
|
+
const val = rawVal.trim();
|
|
103
|
+
if (f.type === 'boolean') {
|
|
104
|
+
const yes = /^(true|yes|1)$/i.test(val); const no = /^(false|no|0)$/i.test(val);
|
|
105
|
+
if (!yes && !no) fail(`--eval "${s}": ${field} is yes/no`);
|
|
106
|
+
return { ...base, step: { node_id: step.id, field, op: op === '!=' ? 'neq' : 'eq', value: yes } };
|
|
107
|
+
}
|
|
108
|
+
if (f.type === 'number') {
|
|
109
|
+
if (!/^-?\d+(\.\d+)?$/.test(val)) fail(`--eval "${s}": ${field} is a number`);
|
|
110
|
+
return { ...base, step: { node_id: step.id, field, op: op === '!=' ? 'neq' : NUM[op], value: Number(val) } };
|
|
111
|
+
}
|
|
112
|
+
if (f.type === 'enum') {
|
|
113
|
+
return { ...base, step: { node_id: step.id, field, op: op === '!=' ? 'not_in' : 'in', value: val.split(',').map((x) => x.trim()) } };
|
|
114
|
+
}
|
|
115
|
+
return { ...base, step: { node_id: step.id, field, op: op === '!=' ? 'neq' : 'eq', value: val } };
|
|
116
|
+
}
|
|
117
|
+
return fail(`--eval "${s}": unknown condition "${cond}". Use failed, passed, not-failed, marked-wrong, not-marked-wrong, marked-right, score<0.5, reason=CODE or STEP.FIELD=value`);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* The flags → { kind, filter, options }. `evals` is GET /evals's list,
|
|
122
|
+
* needed only when --eval names one.
|
|
123
|
+
*/
|
|
124
|
+
function specFromFlags(flags, { kind = flags.kind || 'runs', evals = [] } = {}) {
|
|
125
|
+
if (!['spans', 'runs', 'conversations', 'verdicts'].includes(kind)) fail('--kind is spans, runs, conversations or verdicts');
|
|
126
|
+
const time = timeOf(flags);
|
|
127
|
+
const agents = list(flags.agent);
|
|
128
|
+
const where = list(flags.where).map(whereChip);
|
|
129
|
+
const conds = list(flags.eval).map((e) => evalCondition(e, evals));
|
|
130
|
+
const options = {};
|
|
131
|
+
if (flags.limit !== undefined) {
|
|
132
|
+
const n = parseInt(flags.limit, 10);
|
|
133
|
+
if (!(n > 0)) fail('--limit must be a positive number');
|
|
134
|
+
options.limit = n;
|
|
135
|
+
}
|
|
136
|
+
if (flags.sample !== undefined) options.sample_pct = Number(flags.sample);
|
|
137
|
+
if (flags['include-trace']) options.include_trace = true;
|
|
138
|
+
|
|
139
|
+
if (kind === 'verdicts') {
|
|
140
|
+
const outcomes = list(flags.outcome).flatMap((o) => o.split(',')).map((o) => o.trim().replace(/-/g, '_')).filter(Boolean);
|
|
141
|
+
const ids = conds.map((c) => c.lineage_id).filter(Boolean);
|
|
142
|
+
const filter = { lineage_ids: ids, outcomes: outcomes.length ? outcomes : ['failed'], time };
|
|
143
|
+
if (flags.step) options.node_id = String(flags.step);
|
|
144
|
+
return { kind, filter, options };
|
|
145
|
+
}
|
|
146
|
+
const modularFilters = [];
|
|
147
|
+
if (agents.length) modularFilters.push({ field: 'agent_name', operator: 'is', value: agents, category: kind === 'spans' ? 'global' : 'agents' });
|
|
148
|
+
modularFilters.push(...where);
|
|
149
|
+
const filter = { modularFilters, time };
|
|
150
|
+
if (conds.length) filter.eval_conditions = conds;
|
|
151
|
+
if (kind === 'spans' && flags['any-span']) filter.span_kind = 'any';
|
|
152
|
+
return { kind, filter, options };
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
const needsEvals = (flags) => list(flags.eval).some((e) => !/^any:/i.test(e));
|
|
156
|
+
|
|
157
|
+
module.exports = { specFromFlags, whereChip, evalCondition, timeOf, needsEvals };
|
package/lib/http.js
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/** The API, over fetch (Node 18+). Every call goes to {url}/api/v1. */
|
|
4
|
+
|
|
5
|
+
class ApiError extends Error {
|
|
6
|
+
constructor(status, body) {
|
|
7
|
+
const msg = (body && (body.message || body.error)) || `HTTP ${status}`;
|
|
8
|
+
super(typeof msg === 'string' ? msg : JSON.stringify(msg));
|
|
9
|
+
this.status = status;
|
|
10
|
+
this.body = body;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function client({ url, key, fetchImpl = globalThis.fetch }) {
|
|
15
|
+
if (!key) {
|
|
16
|
+
const e = new Error('No API key. Run `trodo login`, or set TRODO_API_KEY (a trodo_mk_ key from Settings → MCP).');
|
|
17
|
+
e.exitCode = 2;
|
|
18
|
+
throw e;
|
|
19
|
+
}
|
|
20
|
+
const base = `${url}/api/v1`;
|
|
21
|
+
const headers = { Authorization: `Bearer ${key}`, 'User-Agent': `trodo-cli/${require('../package.json').version}`, Accept: 'application/json' };
|
|
22
|
+
|
|
23
|
+
async function request(method, p, { query, body } = {}) {
|
|
24
|
+
const qs = query ? `?${new URLSearchParams(Object.entries(query).filter(([, v]) => v !== undefined && v !== null && v !== '').map(([k, v]) => [k, String(v)]))}` : '';
|
|
25
|
+
const res = await fetchImpl(`${base}${p}${qs}`, {
|
|
26
|
+
method,
|
|
27
|
+
headers: body !== undefined ? { ...headers, 'Content-Type': 'application/json' } : headers,
|
|
28
|
+
body: body !== undefined ? JSON.stringify(body) : undefined,
|
|
29
|
+
});
|
|
30
|
+
return res;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
async function json(method, p, opts) {
|
|
34
|
+
const res = await request(method, p, opts);
|
|
35
|
+
const text = await res.text();
|
|
36
|
+
let data = null;
|
|
37
|
+
try { data = text ? JSON.parse(text) : null; } catch { data = { message: text.slice(0, 300) }; }
|
|
38
|
+
if (!res.ok) throw new ApiError(res.status, data);
|
|
39
|
+
return data;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** NDJSON lines from a response, one parsed object at a time. */
|
|
43
|
+
async function* lines(method, p, opts) {
|
|
44
|
+
const res = await request(method, p, opts);
|
|
45
|
+
if (!res.ok) {
|
|
46
|
+
const text = await res.text();
|
|
47
|
+
let data; try { data = JSON.parse(text); } catch { data = { message: text.slice(0, 300) }; }
|
|
48
|
+
throw new ApiError(res.status, data);
|
|
49
|
+
}
|
|
50
|
+
const decoder = new TextDecoder();
|
|
51
|
+
let buf = '';
|
|
52
|
+
for await (const chunk of res.body) {
|
|
53
|
+
buf += decoder.decode(chunk, { stream: true });
|
|
54
|
+
let i;
|
|
55
|
+
while ((i = buf.indexOf('\n')) >= 0) {
|
|
56
|
+
const line = buf.slice(0, i).trim();
|
|
57
|
+
buf = buf.slice(i + 1);
|
|
58
|
+
if (line) yield JSON.parse(line);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
if (buf.trim()) yield JSON.parse(buf.trim());
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
return {
|
|
65
|
+
get: (p, query) => json('GET', p, { query }),
|
|
66
|
+
post: (p, body, query) => json('POST', p, { body, query }),
|
|
67
|
+
patch: (p, body) => json('PATCH', p, { body }),
|
|
68
|
+
del: (p, query) => json('DELETE', p, { query }),
|
|
69
|
+
lines,
|
|
70
|
+
raw: request,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
module.exports = { client, ApiError };
|
package/lib/main.js
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* trodo — Trodo from a terminal, a script, CI or a coding agent.
|
|
5
|
+
* Every command prints JSON (pretty on a terminal) unless --table is given.
|
|
6
|
+
*/
|
|
7
|
+
const { parse } = require('./args');
|
|
8
|
+
const { ApiError } = require('./http');
|
|
9
|
+
|
|
10
|
+
const pkg = require('../package.json');
|
|
11
|
+
|
|
12
|
+
const HELP = `trodo ${pkg.version} — traces, evals, datasets, prompts and agents from the terminal
|
|
13
|
+
|
|
14
|
+
trodo login [--url https://api.trodo.ai] [--no-browser] sign in through the app
|
|
15
|
+
trodo login --key trodo_mk_… or with a key (CI)
|
|
16
|
+
trodo whoami
|
|
17
|
+
trodo teams the teams this terminal is signed in to
|
|
18
|
+
trodo switch TEAM use another one (login adds teams; --team T for one command)
|
|
19
|
+
trodo logout [TEAM] [--all]
|
|
20
|
+
|
|
21
|
+
Traces
|
|
22
|
+
trodo traces count [filter] how many runs / spans / conversations / verdicts match
|
|
23
|
+
trodo traces stats [filter] [--compare] errors, latency, tokens, cost, feedback, evals, by agent
|
|
24
|
+
trodo traces agents | fields | values FIELD what --agent and --where can name
|
|
25
|
+
trodo traces export [filter] [--include-trace] [--format trodo|openai|dpo] [--target output|expected]
|
|
26
|
+
[-o file | --dir DIR] JSONL, or one JSON file per run / span / conversation
|
|
27
|
+
trodo runs list [filter] [--limit 50] [--offset N]
|
|
28
|
+
trodo runs search TEXT runs whose input, output or spans mention TEXT
|
|
29
|
+
trodo runs get RUN_ID [--tree] the run, spans, messages, feedback, evaluations
|
|
30
|
+
trodo runs tree RUN_ID the span tree
|
|
31
|
+
trodo runs feedback RUN_ID [--thumbs up|down] [--rating 1-5] [--comment TEXT]
|
|
32
|
+
trodo spans list [filter] | search TEXT | get SPAN_ID
|
|
33
|
+
trodo conversations list [filter] [--search TEXT] | get ID [--messages] (alias: threads)
|
|
34
|
+
trodo users get DISTINCT_ID one user's runs, cost, evals and feedback
|
|
35
|
+
|
|
36
|
+
Evals
|
|
37
|
+
trodo evals list | get EVAL names, steps (STEP.FIELD), 30-day stats
|
|
38
|
+
trodo evals steps EVAL [--since 30d] [--table] the paths verdicts took through the steps, with outcomes
|
|
39
|
+
trodo evals verdicts EVAL [--failed|--passed] [--since 7d] [--version N]
|
|
40
|
+
trodo evals verdicts --run ID | --span ID | --conversation ID
|
|
41
|
+
trodo evals verdict VERDICT_ID every step and why
|
|
42
|
+
trodo evals mark VERDICT_ID --right | --wrong --should pass|fail --reason TEXT
|
|
43
|
+
trodo evals queue [--eval EVAL] | review ITEM_ID | grade ITEM_ID CHOICE verdicts waiting on a person
|
|
44
|
+
|
|
45
|
+
Datasets
|
|
46
|
+
trodo datasets list | get DATASET | versions DATASET
|
|
47
|
+
trodo datasets create NAME [-f rows.jsonl] [--kind runs|spans|conversations|verdicts [filter] [--live] [--include-trace]]
|
|
48
|
+
trodo datasets add DATASET --kind KIND [filter] [--live] [--include-trace]
|
|
49
|
+
trodo datasets append DATASET [-f rows.jsonl] {input, output?, expected?} or {run_id}/{span_id}/{conversation_id} lines
|
|
50
|
+
trodo datasets update DATASET [-f updates.jsonl] {id, expected?, output?, metadata?} lines, one version
|
|
51
|
+
trodo datasets export DATASET [--rows passed|failed|expected|no-expected|marked-wrong|all]
|
|
52
|
+
[--eval NAME] [--min-score N] [--format trodo|openai|dpo] [--target ...] [--version N] [-o file]
|
|
53
|
+
|
|
54
|
+
Prompts
|
|
55
|
+
trodo prompts list | get PROMPT [--version N | --label L] | versions PROMPT
|
|
56
|
+
trodo prompts pull PROMPT [--label production] [-o prompt.json]
|
|
57
|
+
trodo prompts push PROMPT -f prompt.json [--message "why"] [--label staging]
|
|
58
|
+
trodo prompts label PROMPT VERSION LABEL [--remove]
|
|
59
|
+
|
|
60
|
+
Issues and agents
|
|
61
|
+
trodo issues list [--status open,regression] | get ID | failures ID | close ID | reopen ID
|
|
62
|
+
trodo agents list | get AGENT | executions AGENT | execution ID
|
|
63
|
+
trodo agents run AGENT [--input JSON] [--wait]
|
|
64
|
+
|
|
65
|
+
trodo api GET|POST|PATCH|DELETE /path [--param k=v] [--data JSON] any /api/v1 route
|
|
66
|
+
trodo setup claude [--global] install the Trodo skills for Claude Code (or --dir DIR)
|
|
67
|
+
|
|
68
|
+
Filter (traces, runs, spans, conversations, datasets create/add):
|
|
69
|
+
--kind spans|runs|conversations|verdicts (traces and datasets; default runs)
|
|
70
|
+
--since 24h|7d|30d|3m|today or --from DATE --to DATE (default 7d)
|
|
71
|
+
--agent NAME repeatable
|
|
72
|
+
--where "status=error" = != ~ > >= < <= on any Traces property:
|
|
73
|
+
total_cost>0.05 span.model~gpt-4o metadata.tier=pro
|
|
74
|
+
--eval "NAME:COND" failed | passed | not-failed | marked-wrong | not-marked-wrong |
|
|
75
|
+
marked-right | score<0.5 | reason=CODE | STEP.FIELD=value | STEP.FIELD<3
|
|
76
|
+
NAME may be "any". Conditions AND together.
|
|
77
|
+
--outcome failed,passed,marked-wrong (verdicts) --step STEP_ID (verdicts, one step per row)
|
|
78
|
+
--limit N --sample PCT
|
|
79
|
+
|
|
80
|
+
Training data:
|
|
81
|
+
trodo traces export --kind spans --eval "any:passed" --eval "any:not-failed" --format openai --target output -o sft.jsonl
|
|
82
|
+
trodo datasets create "Good refunds" --kind runs --eval "Refund policy:passed" --include-trace
|
|
83
|
+
trodo datasets export "Good refunds" --rows passed --format openai --target output -o train.jsonl
|
|
84
|
+
firectl dataset create good-refunds train.jsonl
|
|
85
|
+
|
|
86
|
+
Auth: TRODO_API_KEY / TRODO_API_URL, or \`trodo login\` (~/.trodo/config.json, one key per team).
|
|
87
|
+
A key is a Trodo API key (Settings → MCP); it acts as the person who made it, on its team, with its scopes.
|
|
88
|
+
The CLI reads everything and only adds: nothing it does deletes, archives or replaces data.
|
|
89
|
+
Output: JSON by default; --table for a table, --csv for CSV; lists take --all [--max-items N]. Exit codes: 0 ok, 1 API error, 2 usage.
|
|
90
|
+
`;
|
|
91
|
+
|
|
92
|
+
const COMMANDS = {
|
|
93
|
+
login: ['auth', 'login'], whoami: ['auth', 'whoami'], logout: ['auth', 'logout'], api: ['api', 'call'], setup: ['setup', 'setup'], teams: ['auth', 'teams'], switch: ['auth', 'switch'],
|
|
94
|
+
};
|
|
95
|
+
const ALIASES = { threads: 'conversations', thread: 'conversations', run: 'runs', span: 'spans', dataset: 'datasets', prompt: 'prompts', issue: 'issues', agent: 'agents', eval: 'evals', trace: 'traces' };
|
|
96
|
+
const GROUPS = {
|
|
97
|
+
evals: ['list', 'get', 'steps', 'verdicts', 'verdict', 'mark', 'queue', 'review', 'grade'],
|
|
98
|
+
traces: ['count', 'stats', 'export', 'agents', 'fields', 'values'],
|
|
99
|
+
runs: ['list', 'search', 'get', 'tree', 'feedback'],
|
|
100
|
+
spans: ['list', 'search', 'get'],
|
|
101
|
+
conversations: ['list', 'get'],
|
|
102
|
+
datasets: ['list', 'get', 'create', 'add', 'append', 'update', 'export', 'rows', 'download', 'versions'],
|
|
103
|
+
prompts: ['list', 'get', 'versions', 'pull', 'push', 'label'],
|
|
104
|
+
issues: ['list', 'get', 'failures', 'close', 'reopen'],
|
|
105
|
+
agents: ['list', 'get', 'run', 'executions', 'execution'],
|
|
106
|
+
users: ['get'],
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
async function main(argv) {
|
|
110
|
+
const { flags, positional } = parse(argv);
|
|
111
|
+
const [cmd0, sub, ...rest] = positional;
|
|
112
|
+
const cmd = ALIASES[cmd0] || cmd0;
|
|
113
|
+
// `trodo --version` alone is the CLI's version; `datasets export X --version 3` is a dataset's.
|
|
114
|
+
if (flags.version === true && !cmd) { process.stdout.write(`${pkg.version}\n`); return 0; }
|
|
115
|
+
if (!cmd || flags.help || cmd === 'help') { process.stdout.write(HELP); return 0; }
|
|
116
|
+
try {
|
|
117
|
+
if (COMMANDS[cmd]) {
|
|
118
|
+
const [mod, fn] = COMMANDS[cmd];
|
|
119
|
+
// eslint-disable-next-line global-require, import/no-dynamic-require
|
|
120
|
+
await require(`./commands/${mod}`)[fn]([sub, ...rest].filter(Boolean), flags);
|
|
121
|
+
return process.exitCode || 0;
|
|
122
|
+
}
|
|
123
|
+
if (!GROUPS[cmd]) throw Object.assign(new Error(`Unknown command "${cmd}". \`trodo help\` lists them.`), { exitCode: 2 });
|
|
124
|
+
const action = sub || 'list';
|
|
125
|
+
if (!GROUPS[cmd].includes(action)) throw Object.assign(new Error(`Unknown: trodo ${cmd} ${action}. Try: ${GROUPS[cmd].map((a) => `trodo ${cmd} ${a}`).join(', ')}`), { exitCode: 2 });
|
|
126
|
+
// eslint-disable-next-line global-require, import/no-dynamic-require
|
|
127
|
+
await require(`./commands/${cmd}`)[action](rest, flags);
|
|
128
|
+
return process.exitCode || 0;
|
|
129
|
+
} catch (e) {
|
|
130
|
+
const msg = e instanceof ApiError ? `${e.message}${e.status ? ` (HTTP ${e.status})` : ''}` : e.message;
|
|
131
|
+
process.stderr.write(`trodo: ${msg}\n`);
|
|
132
|
+
if (flags.json) process.stdout.write(`${JSON.stringify({ error: msg, status: e.status || null, detail: e.body || null })}\n`);
|
|
133
|
+
return e.exitCode || 1;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
module.exports = { main, HELP };
|
package/lib/output.js
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* JSON by default (pretty on a terminal, one line when piped) so a coding
|
|
5
|
+
* agent can read every command; --table for a person.
|
|
6
|
+
*/
|
|
7
|
+
const fs = require('fs');
|
|
8
|
+
|
|
9
|
+
function print(data, flags = {}, columns = null) {
|
|
10
|
+
if (flags.csv && Array.isArray(data)) return process.stdout.write(csv(data, columns));
|
|
11
|
+
if (flags.table && columns && Array.isArray(data)) return process.stdout.write(table(data, columns));
|
|
12
|
+
const pretty = process.stdout.isTTY && !flags.json;
|
|
13
|
+
return process.stdout.write(`${JSON.stringify(data, null, pretty ? 2 : 0)}\n`);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function table(rows, columns) {
|
|
17
|
+
const cell = (v) => (v == null ? '' : typeof v === 'object' ? JSON.stringify(v) : String(v)).replace(/\s+/g, ' ');
|
|
18
|
+
const cut = (s, w) => (s.length > w ? `${s.slice(0, w - 1)}…` : s);
|
|
19
|
+
const widths = columns.map(([key, label, max = 40]) => Math.min(max, Math.max(label.length, ...rows.map((r) => cell(r[key]).length))));
|
|
20
|
+
const line = (vals) => vals.map((v, i) => cut(v, widths[i]).padEnd(widths[i])).join(' ').trimEnd();
|
|
21
|
+
return `${[line(columns.map(([, label]) => label)), ...rows.map((r) => line(columns.map(([key]) => cell(r[key]))))].join('\n')}\n`;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** RFC 4180 CSV: the table's columns when the command has them, every key otherwise. */
|
|
25
|
+
function csv(rows, columns) {
|
|
26
|
+
const keys = columns ? columns.map(([k]) => k) : [...new Set(rows.flatMap((r) => (r && typeof r === 'object' ? Object.keys(r) : [])))];
|
|
27
|
+
const cell = (v) => {
|
|
28
|
+
const s = v == null ? '' : typeof v === 'object' ? JSON.stringify(v) : String(v);
|
|
29
|
+
return /[",\r\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
|
|
30
|
+
};
|
|
31
|
+
return `${[keys.join(','), ...rows.map((r) => keys.map((k) => cell(r[k])).join(','))].join('\n')}\n`;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Write lines to --out or stdout; returns { write(obj), close() }. */
|
|
35
|
+
function sink(out) {
|
|
36
|
+
const stream = out ? fs.createWriteStream(out) : process.stdout;
|
|
37
|
+
return {
|
|
38
|
+
write: (obj) => new Promise((resolve) => { if (stream.write(`${JSON.stringify(obj)}\n`)) resolve(); else stream.once('drain', resolve); }),
|
|
39
|
+
close: () => new Promise((resolve) => { if (out) stream.end(resolve); else resolve(); }),
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** Read JSON lines (or one JSON array) from a file or stdin. */
|
|
44
|
+
async function readRows(file) {
|
|
45
|
+
let text;
|
|
46
|
+
if (file && file !== '-') text = fs.readFileSync(file, 'utf8');
|
|
47
|
+
else {
|
|
48
|
+
if (process.stdin.isTTY) { const e = new Error('Give rows with -f rows.jsonl, or pipe them in.'); e.exitCode = 2; throw e; }
|
|
49
|
+
const chunks = [];
|
|
50
|
+
for await (const c of process.stdin) chunks.push(c);
|
|
51
|
+
text = Buffer.concat(chunks).toString('utf8');
|
|
52
|
+
}
|
|
53
|
+
const t = text.trim();
|
|
54
|
+
if (!t) return [];
|
|
55
|
+
if (t.startsWith('[')) return JSON.parse(t);
|
|
56
|
+
return t.split('\n').map((l) => l.trim()).filter(Boolean).map((l, i) => {
|
|
57
|
+
try { return JSON.parse(l); } catch (e) { const err = new Error(`line ${i + 1} is not JSON: ${e.message}`); err.exitCode = 2; throw err; }
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
module.exports = { print, table, csv, sink, readRows };
|
package/package.json
CHANGED
|
@@ -1,6 +1,31 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "trodo-cli",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.3.0",
|
|
4
|
+
"description": "Trodo from the terminal: traces, evals, datasets, prompts, issues and agents for people, scripts and coding agents",
|
|
5
|
+
"bin": {
|
|
6
|
+
"trodo": "bin/trodo.js"
|
|
7
|
+
},
|
|
8
|
+
"main": "./lib/main.js",
|
|
9
|
+
"files": [
|
|
10
|
+
"bin/",
|
|
11
|
+
"lib/",
|
|
12
|
+
"skills/",
|
|
13
|
+
"README.md"
|
|
14
|
+
],
|
|
15
|
+
"scripts": {
|
|
16
|
+
"test": "node --test test/"
|
|
17
|
+
},
|
|
18
|
+
"engines": {
|
|
19
|
+
"node": ">=18"
|
|
20
|
+
},
|
|
21
|
+
"keywords": [
|
|
22
|
+
"trodo",
|
|
23
|
+
"traces",
|
|
24
|
+
"datasets",
|
|
25
|
+
"fine-tuning",
|
|
26
|
+
"evals",
|
|
27
|
+
"cli",
|
|
28
|
+
"agents"
|
|
29
|
+
],
|
|
30
|
+
"license": "ISC"
|
|
31
|
+
}
|