trodo-cli 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -9
- package/lib/args.js +1 -1
- package/lib/commands/agents.js +8 -1
- package/lib/commands/api.js +37 -0
- package/lib/commands/auth.js +67 -9
- package/lib/commands/conversations.js +49 -0
- package/lib/commands/datasets.js +12 -4
- package/lib/commands/evals.js +265 -1
- package/lib/commands/issues.js +40 -0
- package/lib/commands/prompts.js +125 -0
- package/lib/commands/runs.js +90 -13
- package/lib/commands/setup.js +34 -0
- package/lib/commands/spans.js +38 -0
- package/lib/commands/traces.js +70 -4
- package/lib/commands/users.js +14 -0
- package/lib/common.js +69 -0
- package/lib/config.js +47 -6
- package/lib/http.js +26 -7
- package/lib/main.js +63 -21
- package/lib/output.js +12 -1
- package/package.json +3 -2
- package/skills/trodo/SKILL.md +138 -0
- package/skills/trodo-datasets/SKILL.md +116 -0
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const config = require('../config');
|
|
5
|
+
const { client } = require('../http');
|
|
6
|
+
const { print } = require('../output');
|
|
7
|
+
const { usage, pick } = require('../common');
|
|
8
|
+
|
|
9
|
+
/** What a prompt version is, without who saved it: what `pull` writes and `push` reads. */
|
|
10
|
+
const CONTENT_KEYS = ['messages', 'model', 'tools', 'response_format', 'variables'];
|
|
11
|
+
|
|
12
|
+
/** A refusal to edit, said as what to do about it. */
|
|
13
|
+
async function editing(prompt, fn) {
|
|
14
|
+
try { return await fn(); } catch (e) {
|
|
15
|
+
if (e && e.status === 403 && e.body && e.body.error === 'forbidden_edit') {
|
|
16
|
+
throw Object.assign(new Error(`You can't change "${prompt}": only its owner and the editors they add can. Ask the owner to add you (prompt page → Editors).`), { exitCode: 1 });
|
|
17
|
+
}
|
|
18
|
+
throw e;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
async function resolvePrompt(api, ref) {
|
|
23
|
+
const { prompts } = await api.get('/prompts');
|
|
24
|
+
const hit = pick(prompts || [], ref, { kind: 'prompt', listCmd: 'trodo prompts list' });
|
|
25
|
+
const { prompt } = await api.get(`/prompts/${hit.id}`);
|
|
26
|
+
return prompt;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** The version a --version N or --label L names; the newest one otherwise. */
|
|
30
|
+
function versionOf(prompt, flags) {
|
|
31
|
+
const versions = prompt.versions || [];
|
|
32
|
+
if (flags.version !== undefined && flags.version !== true) {
|
|
33
|
+
const v = versions.find((x) => String(x.version_no) === String(flags.version));
|
|
34
|
+
if (!v) throw usage(`"${prompt.name}" has no version ${flags.version}. \`trodo prompts versions "${prompt.name}"\` lists them.`);
|
|
35
|
+
return v;
|
|
36
|
+
}
|
|
37
|
+
if (flags.label !== undefined && flags.label !== true) {
|
|
38
|
+
const v = versions.find((x) => (x.labels || []).includes(String(flags.label)));
|
|
39
|
+
if (!v) throw usage(`No version of "${prompt.name}" has the label "${flags.label}".`);
|
|
40
|
+
return v;
|
|
41
|
+
}
|
|
42
|
+
return versions.reduce((a, b) => (!a || Number(b.version_no) > Number(a.version_no) ? b : a), null);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
async function list(_a, flags) {
|
|
46
|
+
const { prompts } = await client(config.resolve(flags)).get('/prompts', { label: flags.label !== true ? flags.label : undefined, tag: flags.tag !== true ? flags.tag : undefined });
|
|
47
|
+
print(prompts || [], flags, [['name', 'Prompt', 36], ['current_version', 'Version', 8], ['labels', 'Labels', 24], ['id', 'Id', 36], ['updated_at', 'Updated', 24]]);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** trodo prompts get PROMPT [--version N | --label production]: one version, whole. */
|
|
51
|
+
async function get([ref], flags) {
|
|
52
|
+
const prompt = await resolvePrompt(client(config.resolve(flags)), ref);
|
|
53
|
+
const v = versionOf(prompt, flags);
|
|
54
|
+
print({ id: prompt.id, name: prompt.name, description: prompt.description, tags: prompt.tags, version: v }, flags);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async function versions([ref], flags) {
|
|
58
|
+
const prompt = await resolvePrompt(client(config.resolve(flags)), ref);
|
|
59
|
+
const rows = (prompt.versions || []).map((v) => ({ version_no: v.version_no, labels: v.labels, commit_message: v.commit_message, created_by: v.created_by_email, created_at: v.created_at, content_hash: v.content_hash }));
|
|
60
|
+
print(rows, flags, [['version_no', 'v', 4], ['labels', 'Labels', 22], ['commit_message', 'Message', 44], ['created_by', 'By', 24], ['created_at', 'At', 24]]);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** trodo prompts pull PROMPT [--version N | --label L] [-o prompt.json]: the content to edit and push back. */
|
|
64
|
+
async function pull([ref], flags) {
|
|
65
|
+
const prompt = await resolvePrompt(client(config.resolve(flags)), ref);
|
|
66
|
+
const v = versionOf(prompt, flags);
|
|
67
|
+
const out = { name: prompt.name, version_no: v.version_no };
|
|
68
|
+
for (const k of CONTENT_KEYS) out[k] = v[k];
|
|
69
|
+
if (flags.out && flags.out !== true) {
|
|
70
|
+
fs.writeFileSync(String(flags.out), `${JSON.stringify(out, null, 2)}\n`);
|
|
71
|
+
process.stderr.write(`${prompt.name} v${v.version_no} → ${flags.out}\n`);
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
print(out, flags);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* trodo prompts push PROMPT -f prompt.json [--message "why"] [--label staging]
|
|
79
|
+
* A new version from a file `pull` wrote (or any { messages, model?, tools?,
|
|
80
|
+
* variables? }). The prompt is made when no prompt has that name. Content equal
|
|
81
|
+
* to the newest version cuts no version.
|
|
82
|
+
*/
|
|
83
|
+
async function push([ref], flags) {
|
|
84
|
+
const file = flags.file;
|
|
85
|
+
if (!file || file === true) throw usage('Usage: trodo prompts push PROMPT -f prompt.json [--message "why"] [--label staging]');
|
|
86
|
+
let content;
|
|
87
|
+
try { content = JSON.parse(fs.readFileSync(String(file), 'utf8')); } catch (e) { throw usage(`${file} is not a JSON prompt: ${e.message}`); }
|
|
88
|
+
if (!Array.isArray(content.messages) || !content.messages.length) throw usage(`${file} has no messages.`);
|
|
89
|
+
const api = client(config.resolve(flags));
|
|
90
|
+
const name = ref && ref !== true ? String(ref) : content.name;
|
|
91
|
+
if (!name) throw usage('Name the prompt.');
|
|
92
|
+
const { prompts } = await api.get('/prompts');
|
|
93
|
+
let target = (prompts || []).find((p) => p.id === name || p.name === name);
|
|
94
|
+
let created = false;
|
|
95
|
+
if (!target) {
|
|
96
|
+
const made = await api.post('/prompts', { name, ...(content.description ? { description: content.description } : {}) });
|
|
97
|
+
target = made.prompt || made;
|
|
98
|
+
created = true;
|
|
99
|
+
}
|
|
100
|
+
const body = { messages: content.messages };
|
|
101
|
+
for (const k of ['model', 'tools', 'response_format', 'variables']) if (content[k] !== undefined) body[k] = content[k];
|
|
102
|
+
if (flags.message && flags.message !== true) body.commit_message = String(flags.message);
|
|
103
|
+
if (flags.label && flags.label !== true) body.labels = [].concat(flags.label).map(String);
|
|
104
|
+
const out = await editing(name, () => api.post(`/prompts/${target.id}/versions`, body));
|
|
105
|
+
const v = out.version || {};
|
|
106
|
+
print({ prompt: name, id: target.id, created, version_no: v.version_no, labels: v.labels, unchanged: Boolean(out.unchanged) }, flags);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* trodo prompts label PROMPT VERSION LABEL [--remove]
|
|
111
|
+
* Moves LABEL (say production) to VERSION. Agents that fetch the prompt by that
|
|
112
|
+
* label run the new version from their next call.
|
|
113
|
+
*/
|
|
114
|
+
async function label([ref, versionNo, name], flags) {
|
|
115
|
+
if (!ref || !versionNo || !name) throw usage('Usage: trodo prompts label PROMPT VERSION LABEL [--remove]');
|
|
116
|
+
const api = client(config.resolve(flags));
|
|
117
|
+
const prompt = await resolvePrompt(api, ref);
|
|
118
|
+
const v = versionOf(prompt, { version: versionNo });
|
|
119
|
+
const out = await editing(prompt.name, () => (flags.remove
|
|
120
|
+
? api.del(`/prompts/${prompt.id}/versions/${v.version_no}/label/${encodeURIComponent(String(name))}`)
|
|
121
|
+
: api.post(`/prompts/${prompt.id}/versions/${v.version_no}/label`, { label: String(name) })));
|
|
122
|
+
print({ prompt: prompt.name, version_no: v.version_no, ...(out.version ? { labels: out.version.labels } : out) }, flags);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
module.exports = { list, get, versions, pull, push, label };
|
package/lib/commands/runs.js
CHANGED
|
@@ -4,9 +4,10 @@ const config = require('../config');
|
|
|
4
4
|
const { client } = require('../http');
|
|
5
5
|
const { print } = require('../output');
|
|
6
6
|
const { buildSpec } = require('./traces');
|
|
7
|
+
const { usage, listQuery, pages, unitVerdicts, int } = require('../common');
|
|
7
8
|
|
|
8
9
|
const DAY = 86400e3;
|
|
9
|
-
/** A filter's time → { time_start, time_end } for the
|
|
10
|
+
/** A filter's time → { time_start, time_end } for the lists. */
|
|
10
11
|
function windowOf(time, now = Date.now()) {
|
|
11
12
|
if (time.from || time.to) return { time_start: time.from || new Date(now - 7 * DAY).toISOString(), time_end: time.to || new Date(now).toISOString() };
|
|
12
13
|
const p = String(time.preset || '7d');
|
|
@@ -19,23 +20,99 @@ function windowOf(time, now = Date.now()) {
|
|
|
19
20
|
return { time_start: new Date(now - n * unit).toISOString(), time_end: new Date(now).toISOString() };
|
|
20
21
|
}
|
|
21
22
|
|
|
22
|
-
/** trodo runs list [filter flags] [--limit 50] */
|
|
23
|
+
/** trodo runs list [filter flags] [--limit 50] [--offset N] */
|
|
23
24
|
async function listRuns(_args, flags) {
|
|
24
25
|
const api = client(config.resolve(flags));
|
|
25
26
|
const spec = await buildSpec(api, flags, 'runs');
|
|
26
|
-
const
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
// The runs list decodes this once more (decodeURIComponent then JSON), as the app sends it.
|
|
30
|
-
if (mf.length) query.modularFilters = encodeURIComponent(JSON.stringify(mf));
|
|
31
|
-
const out = await api.get('/runs', query);
|
|
32
|
-
print(out.runs || out, flags, [['run_id', 'Run', 36], ['agent_name', 'Agent', 30], ['status', 'Status', 8], ['started_at', 'Started', 24], ['duration_ms', 'ms', 8]]);
|
|
27
|
+
const q = listQuery(spec, windowOf);
|
|
28
|
+
const rows = await pages(flags, async (limit, offset) => { const out = await api.get('/runs', { ...q, limit, offset }); return out.runs || []; }, { max: 100 });
|
|
29
|
+
print(rows, flags, [['run_id', 'Run', 36], ['agent_name', 'Agent', 30], ['status', 'Status', 8], ['started_at', 'Started', 24], ['duration_ms', 'ms', 8]]);
|
|
33
30
|
}
|
|
34
31
|
|
|
35
|
-
/**
|
|
32
|
+
/** The spans of a run as an indented tree, one line each: what a person reads first. */
|
|
33
|
+
function spanTree(spans) {
|
|
34
|
+
const byId = new Map(spans.map((s) => [s.span_id, s]));
|
|
35
|
+
const kids = new Map();
|
|
36
|
+
const roots = [];
|
|
37
|
+
const order = (a, b) => String(a.started_at || '').localeCompare(String(b.started_at || ''));
|
|
38
|
+
for (const s of [...spans].sort(order)) {
|
|
39
|
+
if (s.parent_span_id && byId.has(s.parent_span_id)) {
|
|
40
|
+
if (!kids.has(s.parent_span_id)) kids.set(s.parent_span_id, []);
|
|
41
|
+
kids.get(s.parent_span_id).push(s);
|
|
42
|
+
} else roots.push(s);
|
|
43
|
+
}
|
|
44
|
+
const lines = [];
|
|
45
|
+
const label = (s) => {
|
|
46
|
+
const bits = [`${s.name || s.tool_name || s.kind}`, `[${s.kind || 'span'}]`];
|
|
47
|
+
if (s.model) bits.push(s.model);
|
|
48
|
+
if (s.duration_ms != null) bits.push(`${(Number(s.duration_ms) / 1000).toFixed(2)}s`);
|
|
49
|
+
const tok = (Number(s.input_tokens) || 0) + (Number(s.output_tokens) || 0);
|
|
50
|
+
if (tok) bits.push(`${tok} tok`);
|
|
51
|
+
if (s.status && s.status !== 'ok') bits.push(`✗ ${s.status}${s.error_message ? `: ${String(s.error_message).slice(0, 80)}` : ''}`);
|
|
52
|
+
bits.push(s.span_id);
|
|
53
|
+
return bits.join(' ');
|
|
54
|
+
};
|
|
55
|
+
const walk = (s, prefix, last, depth) => {
|
|
56
|
+
lines.push(`${depth ? `${prefix}${last ? '└─ ' : '├─ '}` : ''}${label(s)}`);
|
|
57
|
+
const c = kids.get(s.span_id) || [];
|
|
58
|
+
c.forEach((k, i) => walk(k, depth ? `${prefix}${last ? ' ' : '│ '}` : '', i === c.length - 1, depth + 1));
|
|
59
|
+
};
|
|
60
|
+
roots.forEach((r, i) => walk(r, '', i === roots.length - 1, 0));
|
|
61
|
+
return lines.join('\n');
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* trodo runs get RUN_ID [--tree]: the run, its spans, messages, feedback and
|
|
66
|
+
* evaluations. --tree prints the span tree instead of JSON.
|
|
67
|
+
*/
|
|
36
68
|
async function getRun([id], flags) {
|
|
37
|
-
if (!id)
|
|
38
|
-
|
|
69
|
+
if (!id) throw usage('Usage: trodo runs get RUN_ID [--tree]');
|
|
70
|
+
const api = client(config.resolve(flags));
|
|
71
|
+
const [detail, verdicts] = await Promise.all([
|
|
72
|
+
api.get(`/runs/${encodeURIComponent(id)}`, { include: 'spans,messages' }),
|
|
73
|
+
api.get(`/eval3/units/run/${encodeURIComponent(id)}/verdicts`).catch(() => null),
|
|
74
|
+
]);
|
|
75
|
+
if (flags.tree) {
|
|
76
|
+
const r = detail.run || {};
|
|
77
|
+
const head = `${r.agent_name || 'run'} ${r.status || ''} ${r.started_at || ''} ${r.duration_ms != null ? `${(r.duration_ms / 1000).toFixed(2)}s` : ''} ${r.run_id || id}`;
|
|
78
|
+
process.stdout.write(`${head}\n${spanTree(detail.spans || [])}\n`);
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
print({ ...detail, evaluations: unitVerdicts(verdicts) }, flags);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** trodo runs tree RUN_ID: the span tree alone. */
|
|
85
|
+
const tree = (args, flags) => getRun(args, { ...flags, tree: true });
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* trodo runs feedback RUN_ID [--thumbs up|down] [--rating 1-5] [--comment TEXT] [--user DISTINCT_ID]
|
|
89
|
+
* A person's feedback on the run, as the SDK and the run page record it.
|
|
90
|
+
*/
|
|
91
|
+
async function feedback([id], flags) {
|
|
92
|
+
if (!id) throw usage('Usage: trodo runs feedback RUN_ID --thumbs up|down [--rating 1-5] [--comment TEXT]');
|
|
93
|
+
const body = {};
|
|
94
|
+
if (flags.thumbs !== undefined) {
|
|
95
|
+
const t = String(flags.thumbs);
|
|
96
|
+
if (!['up', 'down'].includes(t)) throw usage('--thumbs is up or down.');
|
|
97
|
+
body.satisfaction = t === 'up' ? 'positive' : 'negative';
|
|
98
|
+
}
|
|
99
|
+
if (flags.rating !== undefined) {
|
|
100
|
+
const n = Number(flags.rating);
|
|
101
|
+
if (!Number.isInteger(n) || n < 1 || n > 5) throw usage('--rating is a whole number from 1 to 5.');
|
|
102
|
+
body.rating = n;
|
|
103
|
+
}
|
|
104
|
+
if (flags.comment !== undefined && flags.comment !== true) body.comment = String(flags.comment);
|
|
105
|
+
if (flags.user !== undefined && flags.user !== true) body.distinct_id = String(flags.user);
|
|
106
|
+
if (!Object.keys(body).some((k) => ['satisfaction', 'rating', 'comment'].includes(k))) throw usage('Give --thumbs, --rating or --comment.');
|
|
107
|
+
print(await client(config.resolve(flags)).post(`/runs/${encodeURIComponent(id)}/feedback`, body), flags);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** trodo runs search TEXT [--limit 50]: runs whose input, output or spans mention TEXT. */
|
|
111
|
+
async function search(words, flags) {
|
|
112
|
+
const q = words.join(' ').trim();
|
|
113
|
+
if (!q) throw usage('Usage: trodo runs search TEXT [--limit 50]');
|
|
114
|
+
const out = await client(config.resolve(flags)).get('/runs/search', { q, limit: int(flags.limit, 50, 200) });
|
|
115
|
+
print(out.runs || out, flags, [['run_id', 'Run', 36], ['agent_name', 'Agent', 30], ['status', 'Status', 8], ['started_at', 'Started', 24], ['distinct_id', 'User', 26]]);
|
|
39
116
|
}
|
|
40
117
|
|
|
41
|
-
module.exports = { list: listRuns, get: getRun, windowOf };
|
|
118
|
+
module.exports = { list: listRuns, get: getRun, tree, feedback, search, windowOf, spanTree };
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const os = require('os');
|
|
5
|
+
const path = require('path');
|
|
6
|
+
const { print } = require('../output');
|
|
7
|
+
const { usage } = require('../common');
|
|
8
|
+
|
|
9
|
+
const SKILLS_DIR = path.join(__dirname, '..', '..', 'skills');
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* trodo setup claude [--global] | trodo setup --dir DIR
|
|
13
|
+
* Installs the skills this CLI ships (trodo: read and act on traces; trodo-datasets:
|
|
14
|
+
* training data) where a coding agent finds them: ./.claude/skills (this project),
|
|
15
|
+
* ~/.claude/skills with --global, or any folder with --dir.
|
|
16
|
+
*/
|
|
17
|
+
async function setup([agent], flags) {
|
|
18
|
+
let root;
|
|
19
|
+
if (flags.dir && flags.dir !== true) root = path.resolve(String(flags.dir));
|
|
20
|
+
else if (agent === 'claude') root = flags.global ? path.join(os.homedir(), '.claude', 'skills') : path.resolve('.claude', 'skills');
|
|
21
|
+
else throw usage('Usage: trodo setup claude [--global] or trodo setup --dir SKILLS_DIR');
|
|
22
|
+
const installed = [];
|
|
23
|
+
for (const name of fs.readdirSync(SKILLS_DIR)) {
|
|
24
|
+
const src = path.join(SKILLS_DIR, name, 'SKILL.md');
|
|
25
|
+
if (!fs.existsSync(src)) continue;
|
|
26
|
+
const dest = path.join(root, name);
|
|
27
|
+
fs.mkdirSync(dest, { recursive: true });
|
|
28
|
+
fs.copyFileSync(src, path.join(dest, 'SKILL.md'));
|
|
29
|
+
installed.push(path.join(dest, 'SKILL.md'));
|
|
30
|
+
}
|
|
31
|
+
print({ installed }, flags);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
module.exports = { setup };
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const config = require('../config');
|
|
4
|
+
const { client } = require('../http');
|
|
5
|
+
const { print } = require('../output');
|
|
6
|
+
const { buildSpec } = require('./traces');
|
|
7
|
+
const { windowOf } = require('./runs');
|
|
8
|
+
const { usage, listQuery, pages, unitVerdicts, int } = require('../common');
|
|
9
|
+
|
|
10
|
+
/** trodo spans list [filter flags] [--limit 50] [--offset N]: newest first. */
|
|
11
|
+
async function list(_args, flags) {
|
|
12
|
+
const api = client(config.resolve(flags));
|
|
13
|
+
const spec = await buildSpec(api, flags, 'spans');
|
|
14
|
+
const q = listQuery(spec, windowOf);
|
|
15
|
+
const rows = await pages(flags, async (limit, offset) => { const out = await api.get('/spans', { ...q, limit, offset }); return out.spans || []; });
|
|
16
|
+
print(rows, flags, [['span_id', 'Span', 36], ['name', 'Name', 28], ['kind', 'Kind', 8], ['model', 'Model', 22], ['status', 'Status', 8], ['duration_ms', 'ms', 8], ['run_id', 'Run', 36]]);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** trodo spans get SPAN_ID: the span with its full input and output, and its evaluations. */
|
|
20
|
+
async function get([id], flags) {
|
|
21
|
+
if (!id) throw usage('Usage: trodo spans get SPAN_ID');
|
|
22
|
+
const api = client(config.resolve(flags));
|
|
23
|
+
const [detail, verdicts] = await Promise.all([
|
|
24
|
+
api.get(`/spans/${encodeURIComponent(id)}`),
|
|
25
|
+
api.get(`/eval3/units/span/${encodeURIComponent(id)}/verdicts`).catch(() => null),
|
|
26
|
+
]);
|
|
27
|
+
print({ ...detail, evaluations: unitVerdicts(verdicts) }, flags);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** trodo spans search TEXT [--limit 50]: spans whose name, input or output mention TEXT. */
|
|
31
|
+
async function search(words, flags) {
|
|
32
|
+
const q = words.join(' ').trim();
|
|
33
|
+
if (!q) throw usage('Usage: trodo spans search TEXT [--limit 50]');
|
|
34
|
+
const out = await client(config.resolve(flags)).get('/spans/search', { q, limit: int(flags.limit, 50, 200) });
|
|
35
|
+
print(out.spans || out, flags, [['span_id', 'Span', 36], ['name', 'Name', 28], ['kind', 'Kind', 8], ['started_at', 'Started', 24], ['run_id', 'Run', 36]]);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
module.exports = { list, get, search };
|
package/lib/commands/traces.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
3
5
|
const config = require('../config');
|
|
4
6
|
const { client } = require('../http');
|
|
5
7
|
const { print, sink } = require('../output');
|
|
@@ -32,15 +34,79 @@ async function exportTraces(_args, flags) {
|
|
|
32
34
|
const body = { ...spec };
|
|
33
35
|
if (flags.format) body.format = String(flags.format);
|
|
34
36
|
if (flags.target) body.target = String(flags.target);
|
|
35
|
-
const
|
|
37
|
+
const dir = flags.dir && flags.dir !== true ? String(flags.dir) : null;
|
|
38
|
+
if (dir && flags.out) throw Object.assign(new Error('Give -o FILE or --dir DIR, not both.'), { exitCode: 2 });
|
|
39
|
+
if (dir) fs.mkdirSync(dir, { recursive: true });
|
|
40
|
+
const out = dir ? null : sink(flags.out && flags.out !== true ? String(flags.out) : null);
|
|
36
41
|
let summary = null;
|
|
42
|
+
let n = 0;
|
|
37
43
|
for await (const line of api.lines('POST', '/traces/export', { body })) {
|
|
38
44
|
if (line && line._trodo) { summary = line._trodo; continue; }
|
|
45
|
+
n += 1;
|
|
46
|
+
if (dir) {
|
|
47
|
+
// One file per unit, named by what it is: run-<id>.json, span-<id>.json, conversation-<id>.json.
|
|
48
|
+
const src = line.source || {};
|
|
49
|
+
const kind = line.unit_kind || (spec.kind === 'eval_results' ? 'verdict' : spec.kind.replace(/s$/, ''));
|
|
50
|
+
const md = line.metadata || {};
|
|
51
|
+
const byKind = { span: src.span_id || md.span_id, conversation: src.conversation_id || md.conversation_id, verdict: md.eval_run_id || src.eval_run_id, run: src.run_id || md.run_id };
|
|
52
|
+
const id = byKind[kind] || src.span_id || src.run_id || src.conversation_id || String(n).padStart(5, '0');
|
|
53
|
+
fs.writeFileSync(path.join(dir, `${kind}-${String(id).replace(/[^\w.-]/g, '_')}.json`), `${JSON.stringify(line, null, 2)}\n`);
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
39
56
|
// eslint-disable-next-line no-await-in-loop
|
|
40
57
|
await out.write(line);
|
|
41
58
|
}
|
|
42
|
-
await out.close();
|
|
43
|
-
if (summary) process.stderr.write(`${summary.rows} ${spec.kind}${summary.skipped ? `, ${summary.skipped} skipped (no input or no target)` : ''}${flags.out ? ` → ${flags.out}` : ''}\n`);
|
|
59
|
+
if (out) await out.close();
|
|
60
|
+
if (summary) process.stderr.write(`${summary.rows} ${spec.kind}${summary.skipped ? `, ${summary.skipped} skipped (no input or no target)` : ''}${flags.out ? ` → ${flags.out}` : dir ? ` → ${dir}/` : ''}\n`);
|
|
44
61
|
}
|
|
45
62
|
|
|
46
|
-
|
|
63
|
+
/**
|
|
64
|
+
* trodo traces stats [filter flags] [--compare]
|
|
65
|
+
* Count, error rate, latency p50/p95/p99, tokens, cost, feedback, eval
|
|
66
|
+
* outcomes and the busiest agents for what the filter matches. --compare puts
|
|
67
|
+
* the window just before it beside it (the last 7 days against the 7 before).
|
|
68
|
+
*/
|
|
69
|
+
async function stats(_args, flags) {
|
|
70
|
+
const api = client(config.resolve(flags));
|
|
71
|
+
const spec = await buildSpec(api, flags, flags.kind || 'runs');
|
|
72
|
+
const current = await api.post('/traces/stats', spec);
|
|
73
|
+
if (!flags.compare) { print(current, flags); return; }
|
|
74
|
+
const from = new Date(current.window.from).getTime();
|
|
75
|
+
const to = new Date(current.window.to).getTime();
|
|
76
|
+
const prevSpec = { ...spec, filter: { ...spec.filter, time: { from: new Date(from - (to - from)).toISOString(), to: new Date(from).toISOString() } } };
|
|
77
|
+
const previous = await api.post('/traces/stats', prevSpec);
|
|
78
|
+
const pct = (a, b) => (b ? Number((((a - b) / b) * 100).toFixed(1)) : null);
|
|
79
|
+
print({
|
|
80
|
+
current,
|
|
81
|
+
previous,
|
|
82
|
+
change_pct: {
|
|
83
|
+
units: pct(current.units, previous.units),
|
|
84
|
+
error_rate: current.error_rate != null && previous.error_rate != null ? Number(((current.error_rate - previous.error_rate) * 100).toFixed(2)) : null,
|
|
85
|
+
p50_ms: pct(current.latency_ms.p50, previous.latency_ms.p50),
|
|
86
|
+
p95_ms: pct(current.latency_ms.p95, previous.latency_ms.p95),
|
|
87
|
+
cost: pct(current.cost, previous.cost),
|
|
88
|
+
},
|
|
89
|
+
}, flags);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** trodo traces agents: the agent names traces carry (what --agent takes), with run counts. */
|
|
93
|
+
async function agents(_a, flags) {
|
|
94
|
+
const out = await client(config.resolve(flags)).get('/runs/agents');
|
|
95
|
+
print(out.agents || out, flags, [['agent_name', 'Agent', 40], ['run_count', 'Runs', 8], ['error_count', 'Errors', 8], ['total_cost', 'Cost', 12], ['last_seen_at', 'Last seen', 24]]);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** trodo traces fields [--agent NAME]: the properties --where can name, run and span, fixed and custom. */
|
|
99
|
+
async function fields(_a, flags) {
|
|
100
|
+
const agent = flags.agent && flags.agent !== true ? [].concat(flags.agent)[0] : undefined;
|
|
101
|
+
print(await client(config.resolve(flags)).get('/runs/property-keys', { agent_name: agent }), flags);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** trodo traces values FIELD [--search TEXT] [--limit 50]: the values one property takes, most common first. */
|
|
105
|
+
async function values([field], flags) {
|
|
106
|
+
if (!field) throw Object.assign(new Error('Usage: trodo traces values FIELD [--search TEXT] (trodo traces fields lists them)'), { exitCode: 2 });
|
|
107
|
+
const agent = flags.agent && flags.agent !== true ? [].concat(flags.agent)[0] : undefined;
|
|
108
|
+
const out = await client(config.resolve(flags)).get('/runs/property-values', { field, q: flags.search !== true ? flags.search : undefined, agent_name: agent, limit: flags.limit });
|
|
109
|
+
print(out.values || out, flags, [['value', 'Value', 50], ['count', 'Count', 10]]);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
module.exports = { count, export: exportTraces, stats, agents, fields, values, buildSpec };
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const config = require('../config');
|
|
4
|
+
const { client } = require('../http');
|
|
5
|
+
const { print } = require('../output');
|
|
6
|
+
const { usage } = require('../common');
|
|
7
|
+
|
|
8
|
+
/** trodo users get DISTINCT_ID [--since 30d]: one user's runs, conversations, cost, evals and feedback, as the user drawer shows. */
|
|
9
|
+
async function get([id], flags) {
|
|
10
|
+
if (!id) throw usage('Usage: trodo users get DISTINCT_ID');
|
|
11
|
+
print(await client(config.resolve(flags)).get(`/runs/users/${encodeURIComponent(id)}/profile`, { period: flags.since !== true ? flags.since : undefined }), flags);
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
module.exports = { get };
|
package/lib/common.js
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/** Small pieces every command uses. */
|
|
4
|
+
|
|
5
|
+
const usage = (msg) => { const e = new Error(msg); e.exitCode = 2; return e; };
|
|
6
|
+
|
|
7
|
+
/** The one named `ref` (an id, an exact name, or a name in any case) out of `all`. */
|
|
8
|
+
function pick(all, ref, { kind, listCmd, nameKey = 'name', idKeys = ['id'] } = {}) {
|
|
9
|
+
if (ref == null || ref === '' || ref === true) throw usage(`Name the ${kind} (its name or id).`);
|
|
10
|
+
const r = String(ref);
|
|
11
|
+
const hit = all.find((x) => idKeys.some((k) => x[k] != null && String(x[k]) === r))
|
|
12
|
+
|| all.find((x) => x[nameKey] === r)
|
|
13
|
+
|| all.find((x) => String(x[nameKey] || '').toLowerCase() === r.toLowerCase());
|
|
14
|
+
if (!hit) throw usage(`No ${kind} "${r}". \`${listCmd}\` shows them.`);
|
|
15
|
+
return hit;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* A filter spec (filters.specFromFlags) as the query the Runs, Spans and
|
|
20
|
+
* Conversations lists read: the time window plus the Traces filter chips,
|
|
21
|
+
* eval conditions included, sent the way the app sends them.
|
|
22
|
+
*/
|
|
23
|
+
function listQuery(spec, windowOf) {
|
|
24
|
+
const mf = [...(spec.filter.modularFilters || [])];
|
|
25
|
+
if (spec.filter.eval_conditions) mf.push({ category: 'evals', field: 'eval_conditions', operator: 'is', value: spec.filter.eval_conditions });
|
|
26
|
+
const q = { ...windowOf(spec.filter.time) };
|
|
27
|
+
// The lists decode this once more (decodeURIComponent then JSON), as the app sends it.
|
|
28
|
+
if (mf.length) q.modularFilters = encodeURIComponent(JSON.stringify(mf));
|
|
29
|
+
return q;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const int = (v, dflt, max) => {
|
|
33
|
+
const n = Number.parseInt(v, 10);
|
|
34
|
+
if (!Number.isFinite(n) || n < 1) return dflt;
|
|
35
|
+
return max ? Math.min(n, max) : n;
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* One page, or with --all every page (`max` at a time, up to --max-items,
|
|
40
|
+
* 10,000 by default): `page(limit, offset)` returns that page's rows.
|
|
41
|
+
*/
|
|
42
|
+
async function pages(flags, page, { dflt = 50, max = 200 } = {}) {
|
|
43
|
+
if (!flags.all) return page(int(flags.limit, dflt, max), flags.offset);
|
|
44
|
+
const cap = int(flags['max-items'], 10000, 1000000);
|
|
45
|
+
const out = [];
|
|
46
|
+
let offset = Number.parseInt(flags.offset, 10) || 0;
|
|
47
|
+
for (;;) {
|
|
48
|
+
// eslint-disable-next-line no-await-in-loop
|
|
49
|
+
const rows = await page(max, offset);
|
|
50
|
+
out.push(...rows);
|
|
51
|
+
if (rows.length < max || out.length >= cap) break;
|
|
52
|
+
offset += rows.length;
|
|
53
|
+
}
|
|
54
|
+
return out.slice(0, cap);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Every eval's verdicts on one unit (GET /eval3/units/:target/:id/verdicts,
|
|
59
|
+
* grouped by eval) as one list, newest first, each carrying its eval's name.
|
|
60
|
+
*/
|
|
61
|
+
function unitVerdicts(out) {
|
|
62
|
+
if (!out) return undefined;
|
|
63
|
+
if (Array.isArray(out.verdicts)) return out.verdicts;
|
|
64
|
+
const rows = [];
|
|
65
|
+
for (const l of out.lineages || []) for (const v of l.verdicts || []) rows.push({ lineage_name: l.name, ...v });
|
|
66
|
+
return rows.sort((a, b) => String(b.created_at || '').localeCompare(String(a.created_at || '')));
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
module.exports = { usage, pick, listQuery, int, pages, unitVerdicts };
|
package/lib/config.js
CHANGED
|
@@ -3,8 +3,13 @@
|
|
|
3
3
|
/**
|
|
4
4
|
* Where the CLI finds its key and server: TRODO_API_KEY / TRODO_API_URL, else
|
|
5
5
|
* ~/.trodo/config.json (written by `trodo login`, mode 600). A key is a Trodo
|
|
6
|
-
* API key (trodo_mk_…, Settings → MCP); it acts as the person who made
|
|
7
|
-
*
|
|
6
|
+
* API key (trodo_mk_…, Settings → MCP); it acts as the person who made it, on
|
|
7
|
+
* its team, with its scopes.
|
|
8
|
+
*
|
|
9
|
+
* A key belongs to one team, so the file keeps one per team signed in to:
|
|
10
|
+
* { "current_team": "12", "teams": { "12": { "name", "api_key", "api_url" }, … } }
|
|
11
|
+
* `trodo switch TEAM` changes the current one; `--team TEAM` picks one for a
|
|
12
|
+
* single command. A file from 0.2 ({ api_key, api_url }) still works.
|
|
8
13
|
*/
|
|
9
14
|
const fs = require('fs');
|
|
10
15
|
const os = require('os');
|
|
@@ -24,14 +29,50 @@ function write(cfg) {
|
|
|
24
29
|
const f = file();
|
|
25
30
|
fs.mkdirSync(path.dirname(f), { recursive: true, mode: 0o700 });
|
|
26
31
|
fs.writeFileSync(f, `${JSON.stringify(cfg, null, 2)}\n`, { mode: 0o600 });
|
|
32
|
+
// `mode` applies only when the file is created: an older file keeps whatever it had.
|
|
33
|
+
try { fs.chmodSync(f, 0o600); } catch { /* not ours to change (a shared mount); the key is still written */ }
|
|
27
34
|
return f;
|
|
28
35
|
}
|
|
29
36
|
|
|
37
|
+
/** The saved teams as a list, the current one marked. */
|
|
38
|
+
function teams(cfg = read()) {
|
|
39
|
+
return Object.entries(cfg.teams || {}).map(([id, t]) => ({ id, name: t.name || id, api_url: t.api_url || cfg.api_url || DEFAULT_URL, current: String(cfg.current_team) === id, api_key: t.api_key }));
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** A saved team by id or name (exact, then any case); null when none. */
|
|
43
|
+
function findTeam(cfg, ref) {
|
|
44
|
+
const all = teams(cfg);
|
|
45
|
+
const r = String(ref);
|
|
46
|
+
return all.find((t) => t.id === r) || all.find((t) => t.name === r) || all.find((t) => t.name.toLowerCase() === r.toLowerCase()) || null;
|
|
47
|
+
}
|
|
48
|
+
|
|
30
49
|
function resolve(flags = {}) {
|
|
31
50
|
const cfg = read();
|
|
32
|
-
const
|
|
33
|
-
const
|
|
34
|
-
|
|
51
|
+
const envUrl = flags.url || process.env.TRODO_API_URL;
|
|
52
|
+
const finish = (key, url) => ({ key, url: String(envUrl || url || cfg.api_url || DEFAULT_URL).replace(/\/+$/, '') });
|
|
53
|
+
if (flags.key && flags.key !== true) return finish(String(flags.key));
|
|
54
|
+
if (process.env.TRODO_API_KEY) return finish(process.env.TRODO_API_KEY);
|
|
55
|
+
if (flags.team && flags.team !== true) {
|
|
56
|
+
const t = findTeam(cfg, flags.team);
|
|
57
|
+
if (!t) {
|
|
58
|
+
const e = new Error(`Not signed in to a team called "${flags.team}". \`trodo teams\` lists yours; \`trodo login\` adds one.`);
|
|
59
|
+
e.exitCode = 2;
|
|
60
|
+
throw e;
|
|
61
|
+
}
|
|
62
|
+
return finish(t.api_key, t.api_url);
|
|
63
|
+
}
|
|
64
|
+
const cur = cfg.teams && cfg.current_team != null ? cfg.teams[String(cfg.current_team)] : null;
|
|
65
|
+
if (cur) return finish(cur.api_key, cur.api_url);
|
|
66
|
+
return finish(cfg.api_key || null);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Save a team's key and make it the current team. */
|
|
70
|
+
function saveTeam({ id, name, key, url }) {
|
|
71
|
+
const cfg = read();
|
|
72
|
+
const t = { ...(cfg.teams || {}) };
|
|
73
|
+
t[String(id)] = { name, api_key: key, api_url: url };
|
|
74
|
+
const next = { api_url: url, current_team: String(id), teams: t };
|
|
75
|
+
return write(next);
|
|
35
76
|
}
|
|
36
77
|
|
|
37
|
-
module.exports = { read, write, resolve, file, DEFAULT_URL };
|
|
78
|
+
module.exports = { read, write, resolve, file, teams, findTeam, saveTeam, DEFAULT_URL };
|
package/lib/http.js
CHANGED
|
@@ -18,16 +18,34 @@ function client({ url, key, fetchImpl = globalThis.fetch }) {
|
|
|
18
18
|
throw e;
|
|
19
19
|
}
|
|
20
20
|
const base = `${url}/api/v1`;
|
|
21
|
-
const headers = { Authorization: `Bearer ${key}`, 'User-Agent':
|
|
21
|
+
const headers = { Authorization: `Bearer ${key}`, 'User-Agent': `trodo-cli/${require('../package.json').version}`, Accept: 'application/json' };
|
|
22
|
+
|
|
23
|
+
// Reads are retried on a dropped connection or a gateway hiccup (502/503/504):
|
|
24
|
+
// a script pulling thousands of rows should not die on one blip. A write is
|
|
25
|
+
// never retried, so it can never happen twice.
|
|
26
|
+
const READ_POSTS = /^\/traces\/(count|stats|export)$/;
|
|
27
|
+
const isRead = (method, p) => method === 'GET' || (method === 'POST' && READ_POSTS.test(p));
|
|
28
|
+
const RETRY_STATUS = new Set([502, 503, 504]);
|
|
29
|
+
const pause = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
22
30
|
|
|
23
31
|
async function request(method, p, { query, body } = {}) {
|
|
24
32
|
const qs = query ? `?${new URLSearchParams(Object.entries(query).filter(([, v]) => v !== undefined && v !== null && v !== '').map(([k, v]) => [k, String(v)]))}` : '';
|
|
25
|
-
const
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
33
|
+
const attempts = isRead(method, p) ? 4 : 1;
|
|
34
|
+
for (let i = 1; ; i += 1) {
|
|
35
|
+
try {
|
|
36
|
+
// eslint-disable-next-line no-await-in-loop
|
|
37
|
+
const res = await fetchImpl(`${base}${p}${qs}`, {
|
|
38
|
+
method,
|
|
39
|
+
headers: body !== undefined ? { ...headers, 'Content-Type': 'application/json' } : headers,
|
|
40
|
+
body: body !== undefined ? JSON.stringify(body) : undefined,
|
|
41
|
+
});
|
|
42
|
+
if (i < attempts && RETRY_STATUS.has(res.status)) { await pause(500 * 2 ** (i - 1)); continue; } // eslint-disable-line no-await-in-loop
|
|
43
|
+
return res;
|
|
44
|
+
} catch (e) {
|
|
45
|
+
if (i >= attempts) throw Object.assign(new Error(`Could not reach ${url} (${e.cause && e.cause.code ? e.cause.code : e.message}).`), { exitCode: 1 });
|
|
46
|
+
await pause(500 * 2 ** (i - 1)); // eslint-disable-line no-await-in-loop
|
|
47
|
+
}
|
|
48
|
+
}
|
|
31
49
|
}
|
|
32
50
|
|
|
33
51
|
async function json(method, p, opts) {
|
|
@@ -65,6 +83,7 @@ function client({ url, key, fetchImpl = globalThis.fetch }) {
|
|
|
65
83
|
get: (p, query) => json('GET', p, { query }),
|
|
66
84
|
post: (p, body, query) => json('POST', p, { body, query }),
|
|
67
85
|
patch: (p, body) => json('PATCH', p, { body }),
|
|
86
|
+
del: (p, query) => json('DELETE', p, { query }),
|
|
68
87
|
lines,
|
|
69
88
|
raw: request,
|
|
70
89
|
};
|