trodo-cli 0.3.0 → 0.3.1

package/lib/commands/evals.js

@@ -135,4 +135,147 @@ async function steps([ref], flags) {
   print({ eval: e.name, steps: e.steps, ...out }, flags);
 }
 
-module.exports = { list: listEvals, get: getEval, verdicts, verdict, mark, queue, review, grade, steps, daysOf };
+/** A route's condition as words: `verdict = frustrated`, `score < 0.5`. */
+function condText(w) {
+  if (!w || typeof w !== 'object') return String(w);
+  if (Array.isArray(w.all)) return w.all.map(condText).join(' and ');
+  if (Array.isArray(w.any)) return w.any.map(condText).join(' or ');
+  const ops = { eq: '=', ne: '!=', neq: '!=', lt: '<', lte: '<=', gt: '>', gte: '>=', in: 'in', contains: 'contains' };
+  const v = Array.isArray(w.value) ? w.value.join(', ') : w.value;
+  return `${w.field} ${ops[w.op] || w.op} ${v}`;
+}
+
+const asObj = (v) => { if (v && typeof v === 'object') return v; try { return JSON.parse(v); } catch { return null; } };
+
+/** What one step does, from its config, in the terms a person writing a filter needs. */
+function stepWork(n) {
+  const c = n.config || {};
+  if (n.instrument === 'python') return { runs: 'python', code: c.source };
+  if (n.instrument === 'decision_model') {
+    const qs = asObj(c.questions) || {};
+    return {
+      runs: 'decision model (Jev)',
+      reads: c.state,
+      questions: Object.entries(qs).map(([field, q]) => ({
+        field,
+        asks: q.instructions,
+        type: q.type,
+        ...(Array.isArray(q.criteria) ? { levels: q.criteria.map((label, i) => ({ value: i, means: label })) } : {}),
+        ...(Array.isArray(q.options) ? { options: q.options } : {}),
+        ...(q.type === 'score' && Array.isArray(q.criteria) ? { note: `A weighted score from 0 to ${q.criteria.length - 1}: each level times its probability (1.8 is mostly "${q.criteria[q.criteria.length - 1]}"). Filter with a threshold, e.g. ${field}<${(q.criteria.length - 1) - 0.5}.` } : {}),
+      })),
+      may_abstain: String(c.allow_abstain) === 'true' || c.allow_abstain === true,
+    };
+  }
+  if (n.instrument === 'llm_judge') {
+    const prompt = asObj(c.prompt) || c.prompt;
+    return { runs: `LLM judge (${c.model || 'model'})`, prompt };
+  }
+  return { runs: n.instrument || n.type, config: c };
+}
+
+/**
+ * trodo evals explain EVAL [--table]
+ * The eval as a coding agent needs it to write filters: what it checks, each
+ * step in order (what it runs, what it asks or computes, the answers it gives
+ * and their options, where each answer leads), the outcomes with pass / fail
+ * and reason codes, 30-day numbers, and ready filters.
+ */
+async function explain([ref], flags) {
+  const api = client(config.resolve(flags));
+  const e = await resolveEval(api, ref);
+  const [def, stats] = await Promise.all([
+    api.get(`/eval3/definitions/${e.id}`),
+    api.get(`/eval3/definitions/${e.id}/stats`).catch(() => null),
+  ]);
+  const L = def.lineage || {};
+  const g = (def.current && def.current.graph) || {};
+  const nodes = g.nodes || {};
+  const nameOf = (id) => (nodes[id] && nodes[id].name) || id;
+  // Steps in the order a unit walks them, from the entry.
+  const order = [];
+  const seen = new Set();
+  const queue = [g.entry].filter(Boolean);
+  while (queue.length) {
+    const id = queue.shift();
+    if (seen.has(id) || !nodes[id]) continue;
+    seen.add(id);
+    order.push(id);
+    for (const edge of nodes[id].edges || []) queue.push(edge.to || edge.default);
+  }
+  for (const id of Object.keys(nodes)) if (!seen.has(id)) order.push(id);
+  const outcomes = order.filter((id) => nodes[id].type === 'verdict').map((id) => {
+    const c = nodes[id].config || {};
+    const sf = asObj(c.score_from);
+    return { id, name: nodes[id].name, passed: String(c.passed) === 'true' || c.passed === true, reason_code: c.reason_code || null, ...(sf ? { score_from: `${nameOf(sf.node_id)}.${sf.field}` } : {}) };
+  });
+  const steps = order.filter((id) => nodes[id].type !== 'verdict').map((id) => {
+    const n = nodes[id];
+    return {
+      id,
+      name: n.name,
+      kind: n.type,
+      ...stepWork(n),
+      answers: Object.entries(n.output || {}).map(([field, f]) => ({ field, type: f.type, ...(f.values ? { options: f.values } : {}), ...(f.min != null ? { min: f.min, max: f.max } : {}) })),
+      next: (n.edges || []).map((edge) => (edge.default ? { otherwise: nameOf(edge.default) } : { when: condText(edge.when), to: nameOf(edge.to) })),
+    };
+  });
+  const t = (stats && stats.totals) || {};
+  const q = (s) => (/\s/.test(s) ? `"${s}"` : s);
+  const firstEnum = steps.flatMap((s) => s.answers.filter((a) => a.options).map((a) => `${s.name}.${a.field}=${a.options[0]}`))[0];
+  // A number worth a threshold: a judgment's weighted score, else a score on a small scale (not a count's open bound).
+  const scored = steps.flatMap((s) => (s.questions || []).filter((qq) => qq.levels).map((qq) => `${s.name}.${qq.field}<${qq.levels.length - 1.5}`));
+  const small = steps.flatMap((s) => s.answers.filter((a) => a.type === 'number' && a.max != null && Number(a.max) <= 10).map((a) => `${s.name}.${a.field}<${(Number(a.max) + Number(a.min || 0)) / 2}`));
+  const firstNum = scored[0] || small[0];
+  const codes = outcomes.filter((o) => o.reason_code).map((o) => o.reason_code);
+  const out = {
+    id: e.id,
+    name: e.name,
+    checks: L.description || null,
+    target: L.target,
+    agent: L.agent_name || null,
+    enabled: L.enabled,
+    production_version: L.production_version,
+    sample_pct: L.sample_pct != null ? Number(L.sample_pct) : null,
+    last_30_days: t.runs != null ? {
+      verdicts: t.runs, passed: t.passed, failed: t.failed, errored: t.errored,
+      pass_rate: t.runs ? Number((t.passed / Math.max(1, t.passed + t.failed)).toFixed(3)) : null,
+      avg_score: t.avg_score != null ? Number(Number(t.avg_score).toFixed(3)) : null,
+    } : null,
+    steps,
+    outcomes,
+    reason_codes: [...new Set(codes)],
+    filters: [
+      `--eval ${q(`${e.name}:failed`)}`,
+      codes[0] ? `--eval ${q(`${e.name}:reason=${codes[0]}`)}` : null,
+      firstEnum ? `--eval ${q(`${e.name}:${firstEnum}`)}` : null,
+      firstNum ? `--eval ${q(`${e.name}:${firstNum}`)}` : null,
+      `--eval ${q(`${e.name}:marked-wrong`)}`,
+    ].filter(Boolean),
+  };
+  if (!flags.table) { print(out, flags); return; }
+  const lines = [];
+  lines.push(`${out.name}  (${out.target}${out.agent ? `, agent ${out.agent}` : ''}, v${out.production_version}${out.enabled ? '' : ', off'})`);
+  if (out.checks) lines.push(`  ${out.checks}`);
+  if (out.last_30_days) lines.push(`  Last 30 days: ${out.last_30_days.verdicts} verdicts, ${out.last_30_days.passed} passed, ${out.last_30_days.failed} failed, pass rate ${(out.last_30_days.pass_rate * 100).toFixed(1)}%, average score ${out.last_30_days.avg_score}`);
+  lines.push('', 'Steps');
+  steps.forEach((s, i) => {
+    lines.push(`  ${i + 1}. ${s.name} [${s.id}] — ${s.runs}`);
+    for (const qq of s.questions || []) {
+      lines.push(`       asks ${qq.field}: ${qq.asks || ''}`);
+      if (qq.levels) lines.push(`         levels: ${qq.levels.map((l) => `${l.value} = ${l.means}`).join('; ')}`);
+      if (qq.options) lines.push(`         options: ${qq.options.join(', ')}`);
+      if (qq.note) lines.push(`         ${qq.note}`);
+    }
+    if (s.prompt) lines.push(`       prompt: ${String(typeof s.prompt === 'string' ? s.prompt : JSON.stringify(s.prompt)).replace(/\s+/g, ' ').slice(0, 220)}…`);
+    if (s.code) lines.push(`       code: ${String(s.code).split('\n').length} lines of Python (--json shows it)`);
+    lines.push(`       answers: ${s.answers.map((a) => `${a.field} (${a.options ? a.options.join('|') : a.type}${a.min != null ? ` ${a.min}..${a.max}` : ''})`).join(', ')}`);
+    for (const nx of s.next) lines.push(`       ${nx.when ? `if ${nx.when} → ${nx.to}` : `then → ${nx.otherwise}`}`);
+  });
+  lines.push('', 'Outcomes');
+  for (const o of outcomes) lines.push(`  ${o.passed ? 'pass' : 'FAIL'}  ${o.name}${o.reason_code ? `  (${o.reason_code})` : ''}`);
+  lines.push('', 'Filters', ...out.filters.map((f) => `  ${f}`));
+  process.stdout.write(`${lines.join('\n')}\n`);
+}
+
+module.exports = { list: listEvals, get: getEval, explain, verdicts, verdict, mark, queue, review, grade, steps, daysOf, condText };

package/lib/http.js

@@ -20,14 +20,32 @@ function client({ url, key, fetchImpl = globalThis.fetch }) {
   const base = `${url}/api/v1`;
   const headers = { Authorization: `Bearer ${key}`, 'User-Agent': `trodo-cli/${require('../package.json').version}`, Accept: 'application/json' };
 
+  // Reads are retried on a dropped connection or a gateway hiccup (502/503/504):
+  // a script pulling thousands of rows should not die on one blip. A write is
+  // never retried, so it can never happen twice.
+  const READ_POSTS = /^\/traces\/(count|stats|export)$/;
+  const isRead = (method, p) => method === 'GET' || (method === 'POST' && READ_POSTS.test(p));
+  const RETRY_STATUS = new Set([502, 503, 504]);
+  const pause = (ms) => new Promise((r) => setTimeout(r, ms));
+
   async function request(method, p, { query, body } = {}) {
     const qs = query ? `?${new URLSearchParams(Object.entries(query).filter(([, v]) => v !== undefined && v !== null && v !== '').map(([k, v]) => [k, String(v)]))}` : '';
-    const res = await fetchImpl(`${base}${p}${qs}`, {
-      method,
-      headers: body !== undefined ? { ...headers, 'Content-Type': 'application/json' } : headers,
-      body: body !== undefined ? JSON.stringify(body) : undefined,
-    });
-    return res;
+    const attempts = isRead(method, p) ? 4 : 1;
+    for (let i = 1; ; i += 1) {
+      try {
+        // eslint-disable-next-line no-await-in-loop
+        const res = await fetchImpl(`${base}${p}${qs}`, {
+          method,
+          headers: body !== undefined ? { ...headers, 'Content-Type': 'application/json' } : headers,
+          body: body !== undefined ? JSON.stringify(body) : undefined,
+        });
+        if (i < attempts && RETRY_STATUS.has(res.status)) { await pause(500 * 2 ** (i - 1)); continue; } // eslint-disable-line no-await-in-loop
+        return res;
+      } catch (e) {
+        if (i >= attempts) throw Object.assign(new Error(`Could not reach ${url} (${e.cause && e.cause.code ? e.cause.code : e.message}).`), { exitCode: 1 });
+        await pause(500 * 2 ** (i - 1)); // eslint-disable-line no-await-in-loop
+      }
+    }
   }
 
   async function json(method, p, opts) {

package/lib/main.js

@@ -35,6 +35,7 @@ Traces
 
 Evals
   trodo evals list | get EVAL                        names, steps (STEP.FIELD), 30-day stats
+  trodo evals explain EVAL [--table]                 what it checks, each step, answers and options, outcomes, filters
   trodo evals steps EVAL [--since 30d] [--table]     the paths verdicts took through the steps, with outcomes
   trodo evals verdicts EVAL [--failed|--passed] [--since 7d] [--version N]
   trodo evals verdicts --run ID | --span ID | --conversation ID
@@ -94,7 +95,7 @@ const COMMANDS = {
 };
 const ALIASES = { threads: 'conversations', thread: 'conversations', run: 'runs', span: 'spans', dataset: 'datasets', prompt: 'prompts', issue: 'issues', agent: 'agents', eval: 'evals', trace: 'traces' };
 const GROUPS = {
-  evals: ['list', 'get', 'steps', 'verdicts', 'verdict', 'mark', 'queue', 'review', 'grade'],
+  evals: ['list', 'get', 'explain', 'steps', 'verdicts', 'verdict', 'mark', 'queue', 'review', 'grade'],
   traces: ['count', 'stats', 'export', 'agents', 'fields', 'values'],
   runs: ['list', 'search', 'get', 'tree', 'feedback'],
   spans: ['list', 'search', 'get'],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "trodo-cli",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "description": "Trodo from the terminal: traces, evals, datasets, prompts, issues and agents for people, scripts and coding agents",
   "bin": {
     "trodo": "bin/trodo.js"

package/skills/trodo/SKILL.md

@@ -32,6 +32,18 @@ Everything is per team. `trodo teams` lists the teams this terminal is signed in
 to, `trodo switch TEAM` changes the current one, `--team TEAM` picks one for a
 single command. Check `trodo whoami` before writing anything.
 
+## Understand the evals first
+
+```bash
+trodo evals list --table
+trodo evals explain "EVAL" --table     # what it checks, each step, answer options, outcomes, working filters
+```
+
+Read `explain` before writing an `--eval` filter. A decision-model question with
+levels answers a weighted score (1.8 is mostly the top level): use a threshold
+such as `STEP.field<1.5`, never `<2`. A filter matches a unit when any of its
+verdicts matches, so a unit re-scored later can appear under both failed and passed.
+
 ## Find what went wrong
 
 ```bash
@@ -96,6 +108,11 @@ trodo evals queue --table ; trodo evals grade ITEM_ID CHOICE
 
 ## Datasets and training data
 
+Do the thinking in your own script: export with the filter, process the JSONL
+locally (each line has input, output, and every verdict with each step's
+answer), then write rows back with `datasets create NAME -f rows.jsonl` or
+`datasets update`. Nothing of yours runs on Trodo.
+
 ```bash
 trodo traces export --kind spans --eval any:passed --eval any:not-failed --format openai -o sft.jsonl
 trodo traces export --kind runs --since 7d --include-trace --dir ./runs     # one JSON file per run