trodo-cli 0.2.0 → 0.3.1

package/lib/main.js

@@ -9,31 +9,65 @@ const { ApiError } = require('./http');
 
 const pkg = require('../package.json');
 
-const HELP = `trodo ${pkg.version} — traces, datasets and agents from the terminal
+const HELP = `trodo ${pkg.version} — traces, evals, datasets, prompts and agents from the terminal
 
   trodo login [--url https://api.trodo.ai] [--no-browser]   sign in through the app
   trodo login --key trodo_mk_…                             or with a key (CI)
-  trodo whoami | logout
+  trodo whoami
+  trodo teams                                              the teams this terminal is signed in to
+  trodo switch TEAM                                        use another one (login adds teams; --team T for one command)
+  trodo logout [TEAM] [--all]
 
-  trodo evals list                                  eval names and their steps, for --eval
-  trodo traces count  [filter]
-  trodo traces export [filter] [--include-trace] [--format trodo|openai|dpo] [--target output|expected] [-o file]
-  trodo runs list [filter] [--limit 50]
-  trodo runs get RUN_ID
+Traces
+  trodo traces count  [filter]                       how many runs / spans / conversations / verdicts match
+  trodo traces stats  [filter] [--compare]           errors, latency, tokens, cost, feedback, evals, by agent
+  trodo traces agents | fields | values FIELD        what --agent and --where can name
+  trodo traces export [filter] [--include-trace] [--format trodo|openai|dpo] [--target output|expected]
+                      [-o file | --dir DIR]          JSONL, or one JSON file per run / span / conversation
+  trodo runs list [filter] [--limit 50] [--offset N]
+  trodo runs search TEXT                             runs whose input, output or spans mention TEXT
+  trodo runs get RUN_ID [--tree]                     the run, spans, messages, feedback, evaluations
+  trodo runs tree RUN_ID                             the span tree
+  trodo runs feedback RUN_ID [--thumbs up|down] [--rating 1-5] [--comment TEXT]
+  trodo spans list [filter] | search TEXT | get SPAN_ID
+  trodo conversations list [filter] [--search TEXT] | get ID [--messages]      (alias: threads)
+  trodo users get DISTINCT_ID                        one user's runs, cost, evals and feedback
 
+Evals
+  trodo evals list | get EVAL                        names, steps (STEP.FIELD), 30-day stats
+  trodo evals explain EVAL [--table]                 what it checks, each step, answers and options, outcomes, filters
+  trodo evals steps EVAL [--since 30d] [--table]     the paths verdicts took through the steps, with outcomes
+  trodo evals verdicts EVAL [--failed|--passed] [--since 7d] [--version N]
+  trodo evals verdicts --run ID | --span ID | --conversation ID
+  trodo evals verdict VERDICT_ID                     every step and why
+  trodo evals mark VERDICT_ID --right | --wrong --should pass|fail --reason TEXT
+  trodo evals queue [--eval EVAL] | review ITEM_ID | grade ITEM_ID CHOICE         verdicts waiting on a person
+
+Datasets
   trodo datasets list | get DATASET | versions DATASET
-  trodo datasets create NAME [--kind runs|spans|conversations|verdicts [filter] [--live] [--include-trace]]
+  trodo datasets create NAME [-f rows.jsonl] [--kind runs|spans|conversations|verdicts [filter] [--live] [--include-trace]]
   trodo datasets add DATASET --kind KIND [filter] [--live] [--include-trace]
-  trodo datasets append DATASET [-f rows.jsonl]     rows, or {run_id}/{span_id}/{conversation_id} lines
+  trodo datasets append DATASET [-f rows.jsonl]     {input, output?, expected?} or {run_id}/{span_id}/{conversation_id} lines
   trodo datasets update DATASET [-f updates.jsonl]  {id, expected?, output?, metadata?} lines, one version
   trodo datasets export DATASET [--rows passed|failed|expected|no-expected|marked-wrong|all]
                         [--eval NAME] [--min-score N] [--format trodo|openai|dpo] [--target ...] [--version N] [-o file]
 
-  trodo agents list | executions AGENT | execution ID
+Prompts
+  trodo prompts list | get PROMPT [--version N | --label L] | versions PROMPT
+  trodo prompts pull PROMPT [--label production] [-o prompt.json]
+  trodo prompts push PROMPT -f prompt.json [--message "why"] [--label staging]
+  trodo prompts label PROMPT VERSION LABEL [--remove]
+
+Issues and agents
+  trodo issues list [--status open,regression] | get ID | failures ID | close ID | reopen ID
+  trodo agents list | get AGENT | executions AGENT | execution ID
   trodo agents run AGENT [--input JSON] [--wait]
 
-Filter (traces, runs, datasets create/add):
-  --kind spans|runs|conversations|verdicts   (default runs)
+  trodo api GET|POST|PATCH|DELETE /path [--param k=v] [--data JSON]           any /api/v1 route
+  trodo setup claude [--global]                      install the Trodo skills for Claude Code (or --dir DIR)
+
+Filter (traces, runs, spans, conversations, datasets create/add):
+  --kind spans|runs|conversations|verdicts   (traces and datasets; default runs)
   --since 24h|7d|30d|3m|today  or  --from DATE --to DATE      (default 7d)
   --agent NAME                               repeatable
   --where "status=error"                     = != ~ > >= < <= on any Traces property:
@@ -50,25 +84,33 @@ Training data:
   trodo datasets export "Good refunds" --rows passed --format openai --target output -o train.jsonl
   firectl dataset create good-refunds train.jsonl
 
-Auth: TRODO_API_KEY / TRODO_API_URL, or \`trodo login\` (~/.trodo/config.json).
+Auth: TRODO_API_KEY / TRODO_API_URL, or \`trodo login\` (~/.trodo/config.json, one key per team).
 A key is a Trodo API key (Settings → MCP); it acts as the person who made it, on its team, with its scopes.
-Output: JSON by default; --table for a table. Exit codes: 0 ok, 1 API error, 2 usage.
+The CLI reads everything and only adds: nothing it does deletes, archives or replaces data.
+Output: JSON by default; --table for a table, --csv for CSV; lists take --all [--max-items N]. Exit codes: 0 ok, 1 API error, 2 usage.
 `;
 
 const COMMANDS = {
-  login: ['auth', 'login'], whoami: ['auth', 'whoami'], logout: ['auth', 'logout'],
+  login: ['auth', 'login'], whoami: ['auth', 'whoami'], logout: ['auth', 'logout'], api: ['api', 'call'], setup: ['setup', 'setup'], teams: ['auth', 'teams'], switch: ['auth', 'switch'],
 };
+const ALIASES = { threads: 'conversations', thread: 'conversations', run: 'runs', span: 'spans', dataset: 'datasets', prompt: 'prompts', issue: 'issues', agent: 'agents', eval: 'evals', trace: 'traces' };
 const GROUPS = {
-  evals: ['list'],
-  traces: ['count', 'export'],
-  runs: ['list', 'get'],
-  datasets: ['list', 'get', 'create', 'add', 'append', 'update', 'export', 'rows', 'versions'],
-  agents: ['list', 'run', 'executions', 'execution'],
+  evals: ['list', 'get', 'explain', 'steps', 'verdicts', 'verdict', 'mark', 'queue', 'review', 'grade'],
+  traces: ['count', 'stats', 'export', 'agents', 'fields', 'values'],
+  runs: ['list', 'search', 'get', 'tree', 'feedback'],
+  spans: ['list', 'search', 'get'],
+  conversations: ['list', 'get'],
+  datasets: ['list', 'get', 'create', 'add', 'append', 'update', 'export', 'rows', 'download', 'versions'],
+  prompts: ['list', 'get', 'versions', 'pull', 'push', 'label'],
+  issues: ['list', 'get', 'failures', 'close', 'reopen'],
+  agents: ['list', 'get', 'run', 'executions', 'execution'],
+  users: ['get'],
 };
 
 async function main(argv) {
   const { flags, positional } = parse(argv);
-  const [cmd, sub, ...rest] = positional;
+  const [cmd0, sub, ...rest] = positional;
+  const cmd = ALIASES[cmd0] || cmd0;
   // `trodo --version` alone is the CLI's version; `datasets export X --version 3` is a dataset's.
   if (flags.version === true && !cmd) { process.stdout.write(`${pkg.version}\n`); return 0; }
   if (!cmd || flags.help || cmd === 'help') { process.stdout.write(HELP); return 0; }

package/lib/output.js

@@ -7,6 +7,7 @@
 const fs = require('fs');
 
 function print(data, flags = {}, columns = null) {
+  if (flags.csv && Array.isArray(data)) return process.stdout.write(csv(data, columns));
   if (flags.table && columns && Array.isArray(data)) return process.stdout.write(table(data, columns));
   const pretty = process.stdout.isTTY && !flags.json;
   return process.stdout.write(`${JSON.stringify(data, null, pretty ? 2 : 0)}\n`);
@@ -20,6 +21,16 @@ function table(rows, columns) {
   return `${[line(columns.map(([, label]) => label)), ...rows.map((r) => line(columns.map(([key]) => cell(r[key]))))].join('\n')}\n`;
 }
 
+/** RFC 4180 CSV: the table's columns when the command has them, every key otherwise. */
+function csv(rows, columns) {
+  const keys = columns ? columns.map(([k]) => k) : [...new Set(rows.flatMap((r) => (r && typeof r === 'object' ? Object.keys(r) : [])))];
+  const cell = (v) => {
+    const s = v == null ? '' : typeof v === 'object' ? JSON.stringify(v) : String(v);
+    return /[",\r\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
+  };
+  return `${[keys.join(','), ...rows.map((r) => keys.map((k) => cell(r[k])).join(','))].join('\n')}\n`;
+}
+
 /** Write lines to --out or stdout; returns { write(obj), close() }. */
 function sink(out) {
   const stream = out ? fs.createWriteStream(out) : process.stdout;
@@ -47,4 +58,4 @@ async function readRows(file) {
   });
 }
 
-module.exports = { print, table, sink, readRows };
+module.exports = { print, table, csv, sink, readRows };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "trodo-cli",
-  "version": "0.2.0",
-  "description": "Trodo from the terminal: pull traces by their eval results, build datasets, run labeling agents, export fine-tuning data",
+  "version": "0.3.1",
+  "description": "Trodo from the terminal: traces, evals, datasets, prompts, issues and agents for people, scripts and coding agents",
   "bin": {
     "trodo": "bin/trodo.js"
   },
@@ -9,6 +9,7 @@
   "files": [
     "bin/",
     "lib/",
+    "skills/",
     "README.md"
   ],
   "scripts": {

package/skills/trodo/SKILL.md

@@ -0,0 +1,138 @@
+---
+name: trodo
+version: 1.0.0
+cli_version: ">=0.3.0"
+last_updated: 2026-11-03
+description: >-
+  Read and act on an AI agent's production traces in Trodo with the `trodo` CLI:
+  find failing runs, read a run's span tree or a whole conversation, see why an
+  eval failed step by step, check traffic stats against last week, triage and
+  close issues, pull and push prompt versions, give feedback, and turn traces
+  into datasets and fine-tuning files. Use for "why did this run fail", "what
+  broke since yesterday", "show me the conversation", "which evals fail most",
+  "fix this Trodo issue", "update the prompt", "build a dataset from my traces".
+---
+
+# Trodo from the terminal
+
+Every command prints JSON (pipe it to `jq`), `--table` for a person, `--csv` for a
+sheet. Exit codes: 0 ok, 1 the API said no, 2 a usage mistake (the message says which).
+
+## Setup
+
+```bash
+npm i -g trodo-cli
+trodo login      # opens the app: check the code, choose the team, Connect
+trodo whoami
+```
+
+Without a browser (CI): `TRODO_API_KEY=trodo_mk_…` from Settings → MCP.
+
+Everything is per team. `trodo teams` lists the teams this terminal is signed in
+to, `trodo switch TEAM` changes the current one, `--team TEAM` picks one for a
+single command. Check `trodo whoami` before writing anything.
+
+## Understand the evals first
+
+```bash
+trodo evals list --table
+trodo evals explain "EVAL" --table     # what it checks, each step, answer options, outcomes, working filters
+```
+
+Read `explain` before writing an `--eval` filter. A decision-model question with
+levels answers a weighted score (1.8 is mostly the top level): use a threshold
+such as `STEP.field<1.5`, never `<2`. A filter matches a unit when any of its
+verdicts matches, so a unit re-scored later can appear under both failed and passed.
+
+## Find what went wrong
+
+```bash
+trodo traces stats --since 24h --compare            # errors, latency, cost, evals vs the day before
+trodo runs list --since 24h --where "status=error" --table
+trodo runs list --eval "any:failed" --agent support --table
+trodo issues list --status open,regression --table
+trodo runs search "refund for order 4821"           # free text over inputs, outputs and spans
+```
+
+The filter is the Traces page's: `--since 24h|7d|30d`, `--agent NAME`,
+`--where "field op value"` (`= != ~ > >= < <=` on any property:
+`total_cost>0.05`, `span.model~gpt-4o`, `metadata.tier=pro`) and
+`--eval "NAME:COND"` (`failed`, `passed`, `score<0.5`, `reason=CODE`,
+`STEP.FIELD=value`; `trodo evals list` names the evals and their steps).
+`trodo traces agents`, `trodo traces fields` and `trodo traces values FIELD`
+say what `--agent` and `--where` can name.
+
+## Read one
+
+```bash
+trodo runs tree RUN_ID                     # the span tree: names, models, time, tokens, errors
+trodo runs get RUN_ID                      # everything: spans, messages, feedback, evaluations
+trodo spans get SPAN_ID                    # one span's full input and output
+trodo conversations get ID --messages      # the thread as user / assistant turns
+trodo evals verdicts --run RUN_ID          # every eval's verdict on it
+trodo evals verdict VERDICT_ID             # each step the eval took and why
+trodo evals steps EVAL --table             # which step decides most failures
+trodo users get DISTINCT_ID                # one user's runs, cost, evals and feedback
+```
+
+Read the deepest failing span before deciding the cause: what failed above it is
+what the user saw, not why.
+
+## Fix and verify
+
+```bash
+trodo issues get ID                        # root cause, evidence, timeline
+trodo issues failures ID                   # recorded inputs to replay against the fix
+trodo traces count --since 1h --eval "NAME:failed"   # after the fix ships: should fall to 0
+trodo issues close ID                      # a new failure reopens it as a regression
+```
+
+## Prompts
+
+```bash
+trodo prompts pull support-triage --label production -o prompt.json
+# edit prompt.json
+trodo prompts push support-triage -f prompt.json --message "why" --label staging
+trodo prompts label support-triage 5 production     # agents fetching by label switch on their next call
+```
+
+Moving `production` changes what live agents run. Do it only when asked.
+
+## Feedback
+
+```bash
+trodo runs feedback RUN_ID --thumbs down --comment "quoted the wrong plan"
+trodo evals mark VERDICT_ID --wrong --should pass --reason "the refund was within policy"
+trodo evals queue --table ; trodo evals grade ITEM_ID CHOICE
+```
+
+## Datasets and training data
+
+Do the thinking in your own script: export with the filter, process the JSONL
+locally (each line has input, output, and every verdict with each step's
+answer), then write rows back with `datasets create NAME -f rows.jsonl` or
+`datasets update`. Nothing of yours runs on Trodo.
+
+```bash
+trodo traces export --kind spans --eval any:passed --eval any:not-failed --format openai -o sft.jsonl
+trodo traces export --kind runs --since 7d --include-trace --dir ./runs     # one JSON file per run
+trodo datasets create "Refund failures" --kind runs --eval "Refund policy:failed" --include-trace
+trodo datasets rows "Refund failures" -o rows.jsonl
+# write better answers: {"id": "...", "expected": "..."} per line
+trodo datasets update "Refund failures" -f updates.jsonl
+trodo datasets export "Refund failures" --rows expected --format openai --target expected -o train.jsonl
+```
+
+`trodo datasets append NAME -f rows.jsonl` takes `{"input": "…", "output": "…", "expected": "…"}`
+lines, or `{"run_id": "…"}` / `{"span_id": "…"}` / `{"conversation_id": "…"}` to copy traffic.
+
+## What the CLI will not do
+
+It reads everything and only adds. Nothing deletes, archives, replaces or
+reverts data, and traces cannot be changed. If a task needs that, tell the
+person to do it in the app.
+
+## Anything else
+
+`trodo api GET /runs/conversations --param q=alice` calls any `/api/v1` route with the
+signed-in key. `trodo help` lists every command.

package/skills/trodo-datasets/SKILL.md

@@ -0,0 +1,116 @@
+---
+name: trodo-datasets
+version: 1.0.0
+cli_version: ">=0.2.0"
+last_updated: 2026-11-03
+description: >-
+  Turn an agent's production traces into datasets and fine-tuning files with the
+  `trodo` CLI: pick traces by what the team's evals said about them (passed,
+  failed, a score, one step's answer), keep whole runs with their span trees,
+  process rows locally, write Expected back, and export OpenAI / Fireworks /
+  Together chat JSONL or DPO pairs. Use for "build a training set from my
+  traces", "export the runs that passed", "label these rows", "make a dataset of
+  failures", "fine-tune on what went well", and for running a Trodo labeling agent.
+---
+
+# Trodo datasets from the terminal
+
+The team's evals already say which traces went well. A supervised fine-tune
+wants exactly those: their own Output is the target, nothing to label. Rows
+that failed need a better answer first (Expected). Work in that order.
+
+## Setup
+
+```bash
+npm i -g trodo-cli
+trodo login          # opens the app: check the code, choose the team, Connect
+trodo whoami
+```
+
+In CI or anywhere without a browser: `TRODO_API_KEY=trodo_mk_…` (Settings → MCP).
+Every command prints JSON; add `--table` to read it yourself.
+
+## 1. Know the evals
+
+```bash
+trodo evals list --table
+```
+
+`--eval "NAME:COND"` takes an eval's name (or any unique part of it, or `any`):
+`failed`, `passed`, `not-failed`, `marked-wrong`, `not-marked-wrong`,
+`marked-right`, `score<0.5`, `reason=CODE`, or one step's answer
+`STEP.FIELD=value` / `STEP.FIELD<3` (STEP is the step's id or the start of its
+name). Several `--eval` flags AND together. "Passed every eval" is
+`--eval any:passed --eval any:not-failed`. People's corrections count: a
+verdict someone marked wrong is read the other way round.
+
+## 2. Look before you pull
+
+```bash
+trodo traces count --kind runs --since 30d --agent support --eval "any:passed" --eval "any:not-failed"
+```
+
+Kinds: `spans` (one model call), `runs` (one task, add `--include-trace` for
+every span), `conversations` (a whole thread), `verdicts` (eval results, for
+training the eval itself). Narrow with `--where "status=error"`,
+`--where "total_cost>0.05"`, `--where "span.model~gpt-4o"`, `--where "metadata.tier=pro"`.
+
+## 3a. Straight to a training file
+
+```bash
+trodo traces export --kind spans --since 30d --eval any:passed --eval any:not-failed \
+  --format openai --target output -o sft.jsonl
+```
+
+## 3b. Or keep it as a dataset (versioned, shared with the team, labelable)
+
+```bash
+trodo datasets create "Refunds that went well" --kind runs --since 30d \
+  --agent support --eval "Refund policy:passed" --include-trace [--live]
+trodo datasets export "Refunds that went well" --rows passed --format openai --target output -o train.jsonl
+```
+
+`--live` keeps adding new matches every 15 minutes.
+
+## 4. Process rows locally, write them back
+
+```bash
+trodo datasets export "Refund failures" > rows.jsonl          # each line has its "id"
+python label.py < rows.jsonl > updates.jsonl                   # your logic: {"id", "expected", "metadata"?}
+trodo datasets update "Refund failures" -f updates.jsonl --label derived --name "Expected from label.py"
+```
+
+One call is one dataset version (restorable in the app). `--label derived`
+records that a rule wrote Expected, not a person. Add rows with
+`trodo datasets append NAME -f rows.jsonl`: plain rows `{input, output?, expected?}`
+or references `{run_id}` / `{span_id}` / `{conversation_id}` that Trodo fills in.
+
+## 5. Or let a Trodo agent label in the product
+
+The dataset page's **Label** menu (or asking Lucid) makes a labeling agent:
+models that must agree, a model with your rules, a stronger model rewriting
+failures, or two models compared. Run one from here:
+
+```bash
+trodo agents list --table
+trodo agents run "Label Expected where models agree" --wait
+```
+
+## 6. Train
+
+```bash
+trodo datasets export "Refunds" --rows expected --format openai --target expected -o train.jsonl
+firectl dataset create refunds train.jsonl
+firectl sftj create --base-model accounts/fireworks/models/qwen3-8b --dataset refunds --output-model refunds-v1
+```
+
+DPO: `--format dpo` writes Expected as preferred and Output as rejected, for
+rows that have both and differ.
+
+## Rules
+
+- Count first, then pull with a `--limit`. Exports stream; use `-o file`.
+- Never write Expected from a guess. Expected is a person's answer or the
+  output of a rule the team agreed on.
+- The key acts as the person who made it, on one team. A 403 `missing_scope`
+  means the key needs that scope; make a new one with `trodo login`.