unbound-cli 1.11.0 → 1.12.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unbound-cli",
-  "version": "1.11.0",
+  "version": "1.12.0",
   "description": "CLI tool for Unbound - AI Gateway management",
   "main": "src/index.js",
   "bin": {

package/src/api.js

@@ -57,11 +57,15 @@ function request(method, path, { body, query, apiKey, baseUrl, timeoutMs } = {})
     'Accept': 'application/json',
   };
 
-  if (body) {
+  const payload = body ? JSON.stringify(body) : null;
+
+  if (payload !== null) {
     headers['Content-Type'] = 'application/json';
+    // Without this Node falls back to Transfer-Encoding: chunked, and Django's
+    // runserver (wsgiref) reads CONTENT_LENGTH only — so a chunked body arrives
+    // empty and every POST looks like it was sent with no fields at all.
+    headers['Content-Length'] = Buffer.byteLength(payload);
   }
-
-  const payload = body ? JSON.stringify(body) : null;
   const transport = url.protocol === 'https:' ? https : http;
 
   return new Promise((resolve, reject) => {

package/src/chartRender.js

@@ -82,6 +82,22 @@ function reconstructRows(chart_option, chart_meta) {
     return sanitizeForTerminal(v);
   };
 
+  // 0) table charts — the builder emits {columns, rows} under `table` rather than
+  // an axis/series pair, so this shape has no overlap with the ones below. Without
+  // it every table (any result with too many series or dimensions to plot) decodes
+  // to nothing and renders as "(no rows to render)".
+  const table = chart_option.table;
+  if (table && Array.isArray(table.columns) && Array.isArray(table.rows)) {
+    const columns = table.columns.map((c) => sanitizeForTerminal(String(c)));
+    const rows = table.rows.slice(0, MAX_RECONSTRUCT_ROWS).map((r) => (
+      Object.fromEntries(columns.map((k, i) => {
+        const raw = Array.isArray(r) ? r[i] : r?.[table.columns[i]];
+        return [k, sanitizeLabel(raw)];
+      }))
+    ));
+    return { columns, rows };
+  }
+
   // 1) dataset.source — preferred when present
   const source = chart_option.dataset?.source;
   if (Array.isArray(source) && source.length > 0) {

package/src/commands/chat.js

@@ -1,8 +1,13 @@
 /**
  * `unbound chat` — interactive session or one-shot wrapper around
- * POST /api/v1/query_builder/chat/. Conversation history lives in memory
- * only; nothing is written to disk. In one-shot mode (-m), `--json` writes
- * the response to stdout and `-o <path>` writes it to a file.
+ * POST /api/v1/query_builder/agent/converse/. Conversation history lives in
+ * memory only; nothing is written to disk. In one-shot mode (-m), `--json`
+ * writes the response to stdout and `-o <path>` writes it to a file.
+ *
+ * A turn is an ordered transcript: the agent interleaves prose and charts, and
+ * both the terminal rendering and the JSON preserve that order. The JSON carries
+ * decoded rows rather than the chart option the wire format uses, so whatever
+ * consumes it — usually a coding agent — gets tabular data it can work with.
  */
 
 const fs = require('fs');
@@ -16,9 +21,10 @@ const { ensureLoggedIn } = require('../auth');
 const chart = require('../chartRender');
 const { sanitizeForTerminal } = chart;
 
-const ENDPOINT = '/api/v1/query_builder/chat/';
+const ENDPOINT = '/api/v1/query_builder/agent/converse/';
+// Above the server's own 300s cap, so a turn is never abandoned while it works.
+const REQUEST_TIMEOUT_MS = 300000;
 const HISTORY_CAP = 20;
-const RECENT_RESULTS_CAP = 5;
 const MAX_MESSAGE_LEN = 4000;
 const MAX_TURN_CONTENT_LEN = 8000;
 const ALLOWED_ROLES = new Set(['user', 'assistant']);
@@ -79,24 +85,64 @@ function safeWriteFile(target, data) {
   return resolved;
 }
 
-function buildBody(message, history, recent) {
-  return {
+function buildBody(message, history, conversationId) {
+  const body = {
     message,
     conversation_history: history.slice(-HISTORY_CAP),
-    recent_results: recent.slice(-RECENT_RESULTS_CAP),
   };
+  if (conversationId) body.conversation_id = conversationId;
+  return body;
+}
+
+// The model gets the words, not the tables. Same flattening the web client uses.
+function flattenBlocks(blocks) {
+  return (blocks || [])
+    .map((b) => (b.type === 'text' ? b.text : `[chart: ${b.title}]`))
+    .filter(Boolean)
+    .join('\n\n');
 }
 
-function pushRecent(recent, query, result) {
-  recent.push({
-    query,
-    view_spec: result.view_spec,
-    result_summary: {
-      row_count: result.row_count,
-      chart_type: result.chart_meta?.chart_type,
-    },
-  });
-  while (recent.length > RECENT_RESULTS_CAP) recent.shift();
+/**
+ * The transcript, in order, with charts decoded to rows.
+ *
+ * This is the CLI's JSON contract. The wire format carries an ECharts option
+ * because the browser renders one; nothing downstream of `--json` wants that, so
+ * charts are reduced to view_spec + columns + rows.
+ */
+function toJsonPayload(resp) {
+  const blocks = Array.isArray(resp?.blocks) ? resp.blocks : [];
+  return {
+    success: resp?.success !== false,
+    // Without it a --json consumer gets exit 1 and no reason.
+    error: resp?.error ?? null,
+    conversation_id: resp?.conversation_id ?? null,
+    log_id: resp?.log_id ?? null,
+    // Ran out of budget mid-answer; the blocks already here are real.
+    truncated: resp?.truncated === true,
+    blocks: blocks.map((b) => {
+      if (b.type === 'text') return { type: 'text', text: b.text };
+      const result = b.result || {};
+      let columns = [];
+      let rows = [];
+      let decoded = true;
+      try {
+        ({ columns, rows } = chart.reconstructRows(result.chart_option, result.chart_meta));
+      } catch {
+        decoded = false;
+      }
+      return {
+        type: 'chart',
+        title: b.title,
+        view_spec: result.view_spec ?? null,
+        chart_type: result.chart_meta?.chart_type ?? null,
+        row_count: result.row_count ?? rows.length,
+        columns,
+        rows,
+        // Empty rows because it would not decode, not because the answer was empty.
+        ...(decoded ? {} : { decode_failed: true }),
+      };
+    }),
+  };
 }
 
 function pushHistory(history, turn) {
@@ -126,51 +172,31 @@ function friendlyChatError(err) {
   return sanitizeForTerminal(err?.message || 'Request failed.');
 }
 
-function renderSuggestionsForHuman(suggestions) {
-  if (!suggestions?.length) return;
+/** Print one chart: its title, a one-line summary, then a body or a table. */
+function renderChartBlock(block) {
+  const result = block.result || {};
   console.log('');
-  output.info('Suggested follow-ups (rerun `unbound chat -m "<your question>"` with one of these):');
-  suggestions.forEach((s, i) => {
-    const label = sanitizeForTerminal(s.label || `Suggestion ${i + 1}`);
-    const query = sanitizeForTerminal(s.query || '');
-    const desc = sanitizeForTerminal(s.description || '');
-    console.log(`  ${i + 1}. ${bold(label)}`);
-    if (query) console.log(`     ${query}`);
-    if (desc) console.log(`     ${dim(desc)}`);
-  });
-}
-
-/**
- * Print the assistant message + (if a chart) a one-line summary and a chart
- * body. Suggestions are NOT printed here — caller decides (REPL picker vs
- * one-shot numbered list).
- */
-function renderResponse(resp) {
-  const intent = resp?.intent;
-  const message = sanitizeForTerminal(resp?.message || '');
+  console.log(bold('  ' + sanitizeForTerminal(block.title || 'Result')));
+  console.log(dim('  ' + sanitizeForTerminal(chart.summarizeChart(result.chart_meta, result.row_count))));
 
-  if (intent === 'invalid') {
-    output.error(message || 'The request could not be processed.');
-    return;
+  let columns = [];
+  let rows = [];
+  let decoded = true;
+  try {
+    ({ columns, rows } = chart.reconstructRows(result.chart_option, result.chart_meta));
+  } catch {
+    decoded = false;
   }
-
-  if (intent === 'conversational' || !resp.result) {
-    if (message) console.log(cyan('\u25cf ') + message);
+  // Not the same as a chart with no rows, and must not read like one.
+  if (!decoded) {
+    console.log(dim('  (could not read this chart — --json still carries its spec)'));
     return;
   }
-
-  if (message) console.log(cyan('\u25cf ') + message);
-  const result = resp.result;
-  const summary = sanitizeForTerminal(chart.summarizeChart(result.chart_meta, result.row_count));
-  console.log(dim('  ' + summary));
-
-  const { columns, rows } = chart.reconstructRows(result.chart_option, result.chart_meta);
   if (!rows.length) {
     console.log(dim('  (no rows to render)'));
     return;
   }
 
-  console.log('');
   const cap = chart.MAX_RENDER_ROWS;
   const renderRows = rows.slice(0, cap);
   const chartType = result.chart_meta?.chart_type;
@@ -201,19 +227,50 @@ function renderResponse(resp) {
   }
 }
 
+/**
+ * Print the turn in the order the agent produced it — prose where it wrote
+ * prose, a chart where it ran one. A turn can hold several of each.
+ */
+function renderResponse(resp) {
+  if (resp?.success === false) {
+    output.error(sanitizeForTerminal(resp?.error || 'The request could not be processed.'));
+    return;
+  }
+
+  const blocks = Array.isArray(resp?.blocks) ? resp.blocks : [];
+  if (!blocks.length) {
+    output.warn('The assistant returned nothing. Try rephrasing the question.');
+    return;
+  }
+
+  for (const block of blocks) {
+    if (block?.type === 'text') {
+      if (block.text) console.log(cyan('\u25cf ') + sanitizeForTerminal(block.text));
+    } else if (block?.type === 'chart') {
+      renderChartBlock(block);
+    }
+  }
+
+  if (resp?.truncated) {
+    console.log('');
+    output.warn('That is as far as it got in one turn. Ask for the rest to continue.');
+  }
+}
+
 async function runOneShot({ message, json, outputPath }) {
-  const body = buildBody(message, [], []);
+  const body = buildBody(message, [], null);
 
   if (json || outputPath) {
     let resp;
     try {
-      resp = await api.post(ENDPOINT, { body });
+      resp = await api.post(ENDPOINT, { body, timeoutMs: REQUEST_TIMEOUT_MS });
     } catch (err) {
       output.error(friendlyChatError(err));
       process.exitCode = 1;
       return;
     }
-    const serialized = JSON.stringify(resp, null, 2) + '\n';
+    const payload = toJsonPayload(resp);
+    const serialized = JSON.stringify(payload, null, 2) + '\n';
     // Write to stdout first (when requested) so a downstream `-o` write failure
     // doesn't eat the response when both flags are combined.
     if (json) {
@@ -229,14 +286,14 @@ async function runOneShot({ message, json, outputPath }) {
         return;
       }
     }
-    if (resp?.intent === 'invalid') process.exitCode = 1;
+    if (!payload.success || !payload.blocks.length) process.exitCode = 1;
     return;
   }
 
   const spin = output.spinner('Thinking...');
   let resp;
   try {
-    resp = await api.post(ENDPOINT, { body });
+    resp = await api.post(ENDPOINT, { body, timeoutMs: REQUEST_TIMEOUT_MS });
     spin.stop();
   } catch (err) {
     spin.fail(friendlyChatError(err));
@@ -245,8 +302,7 @@ async function runOneShot({ message, json, outputPath }) {
   }
 
   renderResponse(resp);
-  renderSuggestionsForHuman(resp?.suggestions);
-  if (resp?.intent === 'invalid') process.exitCode = 1;
+  if (resp?.success === false || !(resp?.blocks || []).length) process.exitCode = 1;
 }
 
 function printBanner() {
@@ -281,19 +337,19 @@ function handleSlash(line, state) {
       return 'continue';
     case '/clear':
       state.history.length = 0;
-      state.recent.length = 0;
-      state.lastResult = null;
+      state.conversationId = null;
+      state.lastPayload = null;
       output.success('Conversation cleared.');
       return 'continue';
     case '/json':
-      if (state.lastResult) console.log(JSON.stringify(state.lastResult, null, 2));
+      if (state.lastPayload) console.log(JSON.stringify(state.lastPayload, null, 2));
       else console.log(dim('(no result yet)'));
       return 'continue';
     case '/export': {
-      if (!state.lastResult) { output.error('No result to export yet. Ask a question first.'); return 'continue'; }
+      if (!state.lastPayload) { output.error('No result to export yet. Ask a question first.'); return 'continue'; }
       if (!argStr) { output.error('Usage: /export <path>'); return 'continue'; }
       try {
-        const resolved = safeWriteFile(argStr, JSON.stringify(state.lastResult, null, 2) + '\n');
+        const resolved = safeWriteFile(argStr, JSON.stringify(state.lastPayload, null, 2) + '\n');
         output.success(`Wrote ${resolved}`);
       } catch (err) {
         output.error(sanitizeForTerminal(err.message));
@@ -306,8 +362,8 @@ function handleSlash(line, state) {
   }
 }
 
-async function runRepl({ noSuggestions }) {
-  const state = { history: [], recent: [], lastResult: null };
+async function runRepl() {
+  const state = { history: [], conversationId: null, lastPayload: null };
   const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
   let closed = false;
   const setClosed = () => { closed = true; };
@@ -315,20 +371,13 @@ async function runRepl({ noSuggestions }) {
   rl.on('close', setClosed);
 
   printBanner();
-  let pending = null;
 
   while (!closed) {
     let msg;
-    if (pending) {
-      msg = pending;
-      pending = null;
-      console.log(`${bold('chat>')} ${dim('(picked) ') + sanitizeForTerminal(msg)}`);
-    } else {
-      try {
-        msg = await rl.question(`${bold('chat>')} `);
-      } catch {
-        break;
-      }
+    try {
+      msg = await rl.question(`${bold('chat>')} `);
+    } catch {
+      break;
     }
     if (closed) break;
     msg = (msg || '').trim();
@@ -347,12 +396,12 @@ async function runRepl({ noSuggestions }) {
     // Build the request body BEFORE pushing the current turn into history —
     // otherwise `message` and the last entry of `conversation_history` would
     // both contain the same user text, making the backend process it twice.
-    const body = buildBody(msg, state.history, state.recent);
+    const body = buildBody(msg, state.history, state.conversationId);
 
     const spin = output.spinner('Thinking...');
     let resp;
     try {
-      resp = await api.post(ENDPOINT, { body });
+      resp = await api.post(ENDPOINT, { body, timeoutMs: REQUEST_TIMEOUT_MS });
       spin.stop();
     } catch (err) {
       spin.fail(friendlyChatError(err));
@@ -360,44 +409,32 @@ async function runRepl({ noSuggestions }) {
       continue;
     }
 
-    // Record the user turn only after a successful round-trip so history stays
-    // consistent with the server view.
-    pushHistory(state.history, { role: 'user', content: msg, timestamp: nowIso() });
+    // No answer means no turn: recording the question would send the assistant
+    // something it never replied to. The id stays so the server's rows group.
+    if (resp?.conversation_id) state.conversationId = resp.conversation_id;
+    if (resp?.success === false || !(resp?.blocks || []).length) {
+      renderResponse(resp);
+      continue;
+    }
 
-    const assistantTurn = normalizeAssistantTurn(resp?.turn);
-    if (assistantTurn) pushHistory(state.history, assistantTurn);
-    if (resp?.result) {
-      pushRecent(state.recent, msg, resp.result);
-      state.lastResult = resp.result;
+    const assistantTurn = normalizeAssistantTurn({
+      role: 'assistant',
+      content: flattenBlocks(resp?.blocks),
+      timestamp: nowIso(),
+    });
+    // The pair goes in together or not at all. A reply that flattens to nothing
+    // would leave the question in history with no answer beside it, and the next
+    // request would send the model two user turns in a row.
+    if (!assistantTurn?.content) {
+      renderResponse(resp);
+      continue;
     }
+    pushHistory(state.history, { role: 'user', content: msg, timestamp: nowIso() });
+    pushHistory(state.history, assistantTurn);
+    state.lastPayload = toJsonPayload(resp);
 
     renderResponse(resp);
 
-    const suggestions = Array.isArray(resp?.suggestions) ? resp.suggestions : [];
-    if (suggestions.length && !noSuggestions && process.stdin.isTTY && process.stdout.isTTY) {
-      const pickOptions = [
-        ...suggestions.map((s, i) => ({
-          label: sanitizeForTerminal(s.label || `Suggestion ${i + 1}`)
-            + (s.description ? ' ' + dim('— ' + sanitizeForTerminal(s.description)) : ''),
-          value: sanitizeForTerminal(s.query || ''),
-        })).filter((o) => o.value),
-        { label: dim('(type your own)'), value: null },
-        { label: dim('(exit)'), value: '__exit__' },
-      ];
-      let pick;
-      try {
-        rl.pause();
-        pick = await output.select('Follow up?', pickOptions);
-      } catch {
-        pick = null;
-      } finally {
-        rl.resume();
-      }
-      if (pick === '__exit__') break;
-      if (pick) pending = pick;
-    } else if (suggestions.length && (!process.stdin.isTTY || !process.stdout.isTTY)) {
-      renderSuggestionsForHuman(suggestions);
-    }
   }
 
   rl.close();
@@ -411,64 +448,54 @@ function register(program) {
     .option('-m, --message <text>', 'Ask a single question and exit (no interactive session).')
     .option('--json', 'Write the raw JSON response to stdout. Use with -m.')
     .option('-o, --output <path>', 'Write the raw JSON response to a file. Refuses to overwrite. Use with -m.')
-    .option('--no-suggestions', 'Skip the follow-up picker after each turn (interactive session only).')
     .addHelpText('after', `
 EXAMPLES
   Start an interactive session:
     $ unbound chat
 
-  Ask a single question and render a chart in the terminal:
+  Ask a single question and render it in the terminal:
     $ unbound chat -m "show cost by provider for the last 30 days"
 
-  Write the raw JSON response to stdout (pipe to jq, a script, or another tool):
+  Write the JSON transcript to stdout (pipe to jq, a script, or a coding agent):
     $ unbound chat -m "cost by provider last 30 days" --json
-    $ unbound chat -m "cost by provider last 30 days" --json | jq '.intent'
-    $ unbound chat -m "cost by provider last 30 days" --json | jq '.result.chart_option'
+    $ unbound chat -m "cost by provider last 30 days" --json | jq '.blocks[].text'
+    $ unbound chat -m "cost by provider last 30 days" --json | jq '.blocks[] | select(.type=="chart") | .rows'
 
-  Write the raw JSON response to a file:
+  Write it to a file:
     $ unbound chat -m "cost by provider last 30 days" -o cost.json
 
   Combine --json and -o to pipe and archive at the same time:
-    $ unbound chat -m "cost by provider last 30 days" --json -o cost.json | jq '.intent'
+    $ unbound chat -m "cost by provider last 30 days" --json -o cost.json | jq '.blocks | length'
 
 OUTPUT FORMAT (--json and -o)
-  { message, intent, result?, suggestions?, turn }
-
-  intent: "sql_visualise" | "conversational" | "invalid"
-    sql_visualise  — a chart was generated; 'result' is populated.
-    conversational — the model wants clarification; pick a suggestions[i].query and re-run.
-    invalid        — the prompt could not be answered; exit code is 1.
-
-  result (only when intent == "sql_visualise"):
-    columns      string[]             left-to-right column names
-    row_count    number               total rows in the answer
-    chart_meta   { chart_type, x, y } chart_type is one of:
-                                        "bar" | "line" | "donut" | "pie"
-                                      | "kpi" | "scatter" | "table"
-    chart_option ECharts config with the data embedded.
-
-  suggestions (may be present with any intent):
-    [{ label, query, description }]
-    Each entry's 'query' is the exact text for a follow-up '-m "<query>"' call.
-
-  turn: { role, content, timestamp } — the assistant's turn record.
-
-DATA EXTRACTION (rows are embedded in chart_option; no separate rows field)
-  bar, line        chart_option.xAxis.data         (category labels, ordered)
-                   chart_option.series[i].data     (values, parallel to xAxis.data)
-  donut, pie       chart_option.series[0].data     ([{name, value}, ...])
-  scatter          chart_option.series[0].data     ([[x, y], ...])
-                   or chart_option.dataset.source
-  table            chart_option.dataset.source     (first row is the header)
-  kpi              chart_option.series[0].data[0]  (a single number; no axes)
+  { success, conversation_id, log_id, truncated, blocks[] }
+
+  A turn is an ordered transcript. The agent answers one thing at a time, so a
+  single question can return several charts with prose between them, and blocks[]
+  preserves that order.
+
+  blocks[i] is one of:
+
+    { type: "text", text }
+      What the agent wrote at this point in the answer.
+
+    { type: "chart", title, view_spec, chart_type, row_count, columns, rows }
+      columns    string[]                   left-to-right column names
+      rows       [{ <column>: value, ... }]  the data, already decoded
+      chart_type "bar" | "line" | "donut" | "pie" | "kpi" | "scatter" | "table"
+      view_spec  the query that produced it, for reuse or refinement
+
+  truncated: true means the agent ran out of budget partway. The blocks present
+  are real and complete; ask a follow-up for the rest.
+
+  log_id identifies the stored turn, for feedback and for support to trace.
 
 EXIT CODES
   0  success
-  1  intent=="invalid", auth/role/network failure, or a write error
+  1  no blocks returned, auth/role/network failure, or a write error
 
 INTERACTIVE SESSION
   $ unbound chat                             start the session
-  $ unbound chat --no-suggestions            skip the follow-up picker
 
   Commands (type inside the session):
     /help              show these commands
@@ -516,7 +543,7 @@ NOTES
       }
 
       try {
-        await runRepl({ noSuggestions: opts.suggestions === false });
+        await runRepl();
       } catch (err) {
         output.error(friendlyChatError(err));
         process.exitCode = 1;
@@ -527,5 +554,8 @@ NOTES
 module.exports = {
   register,
   // Exposed for unit tests:
-  _internals: { safeWriteFile, expandPath, normalizeAssistantTurn, friendlyChatError },
+  _internals: {
+    safeWriteFile, expandPath, normalizeAssistantTurn, friendlyChatError,
+    toJsonPayload, flattenBlocks, buildBody,
+  },
 };

package/src/commands/setup.js

@@ -209,11 +209,15 @@ async function runPythonScript(scriptPath, args, { capture }) {
         windowsHide: true,
       });
       let out = '';
+      // stderr is also kept on its own. Python block-buffers stdout when piped,
+      // so a crash's stdout can flush after the traceback and interleave with it
+      // in `out` — failureReason needs an uncontaminated stream to read.
+      let errOut = '';
       if (capture) {
         child.stdin.on('error', () => {});
         child.stdin.end();
         child.stdout.on('data', (d) => { out += d.toString(); });
-        child.stderr.on('data', (d) => { out += d.toString(); });
+        child.stderr.on('data', (d) => { const s = d.toString(); out += s; errOut += s; });
       }
       child.on('close', (code) => {
         if (code === 0) return resolve();
@@ -223,7 +227,10 @@ async function runPythonScript(scriptPath, args, { capture }) {
         // Carry the numeric exit code so callers can report it to the failure
         // ledger (reportSetupFailure) without re-parsing the message.
         err.exitCode = code;
-        if (capture) err.setupOutput = out.trim();
+        if (capture) {
+          err.setupOutput = out.trim();
+          err.stderr = errOut.trim();
+        }
         reject(err);
       });
       child.on('error', reject);
@@ -576,6 +583,61 @@ async function runBatch(tools, runFn, { clear = false, summary = null, report =
   return { ok: true, skipped: mdmSkipped };
 }
 
+/**
+ * Picks the most useful one-line reason out of a failed setup script's captured
+ * output. The python scripts open AND close with a `====` banner, so the first
+ * line is never the reason a step failed — taking it verbatim produced the
+ * useless `Codex (gateway) (====================)` that hid a real bug. Prefer
+ * the last line the script itself flagged as an error, then the exit code.
+ *
+ * Tolerates a non-Error throw: this runs inside nuke's per-tool catch, so
+ * throwing here would abort the remaining tools instead of clearing them.
+ */
+function failureReason(err) {
+  err = err || {};
+  // Read the traceback from stderr when we have it: `out` merges both streams by
+  // arrival order, so buffered stdout can land between the header and the
+  // exception. Fall back to the merged text for callers that don't set stderr.
+  const crash = exceptionLine(err.stderr) || exceptionLine(err.message);
+  if (crash) return truncate(crash);
+  const lines = (err.message || '').split('\n').map((l) => l.trim()).filter(Boolean);
+  // Otherwise the lines the scripts themselves flag. The dotted form catches a
+  // bare exception line captured without its traceback header.
+  const flagged = lines.filter((l) => /^(❌|failed\b|error\b|exception\b|[\w.]*(error|exception):)/i.test(l));
+  const pick = flagged.length ? flagged[flagged.length - 1] : null;
+  if (pick) return truncate(pick);
+  if (err.exitCode != null) return `exit code ${err.exitCode}`;
+  return truncate(lines[0] || 'failed');
+}
+
+/**
+ * The exception a python traceback ended on, or null if `text` has no traceback.
+ * Structural rather than positional: frame lines are indented, so the exception
+ * is the first UNindented line after the header. That holds even if buffered
+ * stdout flushed into the middle of the traceback. The LAST header wins, since
+ * with chained exceptions ("During handling of the above exception...") the final
+ * one is what actually propagated.
+ */
+function exceptionLine(text) {
+  const raw = (text || '').split('\n');
+  let header = -1;
+  for (let i = 0; i < raw.length; i++) {
+    if (/^traceback\b/i.test(raw[i].trim())) header = i;
+  }
+  if (header === -1) return null;
+  const exc = raw.slice(header + 1).find(
+    (l) => l.trim() && !/^\s/.test(l) && !/^=+$/.test(l.trim())
+  );
+  return exc ? exc.trim() : null;
+}
+
+// Slice by code point, not code unit, so an 80-char cut can't split an emoji
+// or other astral character into a broken fragment.
+function truncate(s, max = 80) {
+  const chars = Array.from(s);
+  return chars.length > max ? chars.slice(0, max).join('') : s;
+}
+
 /**
  * Clears a set of tools best-effort: a single tool failing does not abort the
  * rest (unlike runBatch, which stops on first failure). Used by `uninstall`,
@@ -592,8 +654,7 @@ async function clearToolsBestEffort(layer, tools, { mdm, backendUrl, gatewayUrl,
       await runScriptPiped(tool.script, args);
       s.succeed(`${layer}: ${tool.label}`);
     } catch (err) {
-      const reason = (err.message || 'failed').split('\n')[0].slice(0, 80);
-      s.fail(`${layer}: ${tool.label} (${reason})`);
+      s.fail(`${layer}: ${tool.label} (${failureReason(err)})`);
       failed.push(tool.label);
     }
   }
@@ -1183,6 +1244,7 @@ module.exports = {
   scriptSupportsBackfill,
   resolveSetupAllTools,
   clearUnboundEnvsEverywhere,
+  failureReason,
   NUKE_ENV_VARS,
   removeBinaryInstall,
   runBinaryClear,

package/test/chart-render.test.js

@@ -203,3 +203,27 @@ test('formatNumber: integer thousands separator, decimal capped', () => {
   assert.equal(chart.formatNumber(12.3456), '12.35');
   assert.equal(chart.formatNumber(0), '0');
 });
+
+test('reconstructRows: decodes a table chart_option', () => {
+  const { columns, rows } = chart.reconstructRows(
+    { table: { columns: ['AI Model', 'Requests'], rows: [{ 'AI Model': 'gpt-4o', Requests: 172 }] } },
+    { chart_type: 'table' },
+  );
+
+  assert.deepEqual(columns, ['AI Model', 'Requests']);
+  assert.deepEqual(rows, [{ 'AI Model': 'gpt-4o', Requests: 172 }]);
+});
+
+test('reconstructRows: table rows may be arrays aligned to columns', () => {
+  const { columns, rows } = chart.reconstructRows(
+    { table: { columns: ['a', 'b'], rows: [['x', 1], ['y', 2]] } },
+    { chart_type: 'table' },
+  );
+
+  assert.deepEqual(columns, ['a', 'b']);
+  assert.deepEqual(rows, [{ a: 'x', b: 1 }, { a: 'y', b: 2 }]);
+});
+
+test('reconstructRows: a malformed table falls through instead of throwing', () => {
+  assert.deepEqual(chart.reconstructRows({ table: { columns: 'nope' } }, {}), { columns: [], rows: [] });
+});

package/test/chat-internals.test.js

@@ -142,3 +142,117 @@ test('friendlyChatError: sanitizes ANSI from backend error messages', () => {
   const msg = friendlyChatError({ statusCode: 500, message: '\x1b]0;title\x07boom\x1b[31m' });
   assert.equal(msg, 'boom');
 });
+
+// --- agent transcript mapping -------------------------------------------------
+
+const { toJsonPayload, flattenBlocks, buildBody } = chat._internals;
+
+// One chart, shaped the way the agent endpoint returns it: a bar chart whose
+// rows live inside the ECharts option rather than in a rows field.
+function chartBlock(title) {
+  return {
+    type: 'chart',
+    title,
+    result: {
+      view_spec: { base_table: 'gateway_metrics_summary' },
+      row_count: 2,
+      chart_meta: { chart_type: 'bar', x: 'dimension', y: ['metric_value'], column_labels: {} },
+      chart_option: {
+        xAxis: { data: ['openai', 'anthropic'] },
+        series: [{ name: 'metric_value', data: [10, 20] }],
+      },
+    },
+  };
+}
+
+test('toJsonPayload: keeps prose and charts in the order the agent produced them', () => {
+  const payload = toJsonPayload({
+    success: true,
+    conversation_id: 'c1',
+    log_id: 'l1',
+    blocks: [
+      { type: 'text', text: 'Here is spend.' },
+      chartBlock('Cost by provider'),
+      { type: 'text', text: 'Note the gap.' },
+    ],
+  });
+
+  assert.deepEqual(payload.blocks.map((b) => b.type), ['text', 'chart', 'text']);
+  assert.equal(payload.conversation_id, 'c1');
+  assert.equal(payload.log_id, 'l1');
+  assert.equal(payload.truncated, false);
+});
+
+test('toJsonPayload: decodes chart rows so consumers never parse an ECharts option', () => {
+  const payload = toJsonPayload({ blocks: [chartBlock('Cost by provider')] });
+  const chartOut = payload.blocks[0];
+
+  assert.equal(chartOut.title, 'Cost by provider');
+  assert.equal(chartOut.chart_type, 'bar');
+  assert.equal(chartOut.row_count, 2);
+  assert.ok(chartOut.columns.length >= 2, 'columns should be populated');
+  assert.equal(chartOut.rows.length, 2);
+  assert.deepEqual(chartOut.view_spec, { base_table: 'gateway_metrics_summary' });
+  // The wire format is an implementation detail of the browser renderer.
+  assert.equal(chartOut.chart_option, undefined);
+});
+
+test('toJsonPayload: an undecodable chart still appears, with its spec intact', () => {
+  const payload = toJsonPayload({
+    blocks: [{ type: 'chart', title: 'Broken', result: { view_spec: { a: 1 }, row_count: 7 } }],
+  });
+
+  assert.equal(payload.blocks.length, 1);
+  assert.equal(payload.blocks[0].row_count, 7);
+  assert.deepEqual(payload.blocks[0].view_spec, { a: 1 });
+  assert.deepEqual(payload.blocks[0].rows, []);
+});
+
+test('toJsonPayload: surfaces truncation and failure rather than hiding them', () => {
+  assert.equal(toJsonPayload({ blocks: [], truncated: true }).truncated, true);
+  assert.equal(toJsonPayload({ success: false, error: 'nope' }).success, false);
+  assert.deepEqual(toJsonPayload(null).blocks, []);
+  assert.deepEqual(toJsonPayload({ blocks: 'not-an-array' }).blocks, []);
+});
+
+test('flattenBlocks: a chart becomes its title, so history carries words only', () => {
+  const flat = flattenBlocks([
+    { type: 'text', text: 'First.' },
+    { type: 'chart', title: 'Cost by provider' },
+    { type: 'text', text: 'Second.' },
+  ]);
+
+  assert.equal(flat, 'First.\n\n[chart: Cost by provider]\n\nSecond.');
+  assert.equal(flattenBlocks(undefined), '');
+});
+
+test('buildBody: omits conversation_id on the first turn, sends it after', () => {
+  assert.equal('conversation_id' in buildBody('hi', [], null), false);
+  assert.equal(buildBody('hi', [], 'c9').conversation_id, 'c9');
+});
+
+test('toJsonPayload: carries the failure reason, not just the failure', () => {
+  const payload = toJsonPayload({ success: false, error: 'The assistant is unavailable right now.' });
+
+  assert.equal(payload.success, false);
+  // Without this a --json consumer gets an exit code and nothing to act on.
+  assert.equal(payload.error, 'The assistant is unavailable right now.');
+  assert.equal(toJsonPayload({ blocks: [] }).error, null);
+});
+
+test('toJsonPayload: an empty chart and an unreadable one are not the same block', () => {
+  const unreadable = { get table() { throw new TypeError('unreadable'); } };
+  const payload = toJsonPayload({
+    blocks: [
+      { type: 'chart', title: 'Broken', result: { chart_option: unreadable, view_spec: { a: 1 }, row_count: 7 } },
+      { type: 'chart', title: 'Empty', result: { chart_option: {}, view_spec: { b: 2 }, row_count: 0 } },
+    ],
+  });
+
+  assert.equal(payload.blocks[0].decode_failed, true);
+  assert.deepEqual(payload.blocks[0].rows, []);
+  // The spec survives, so the caller still knows what was asked.
+  assert.deepEqual(payload.blocks[0].view_spec, { a: 1 });
+  // A genuinely empty answer carries no such marker.
+  assert.equal('decode_failed' in payload.blocks[1], false);
+});

package/test/setup-args.test.js

@@ -3,7 +3,7 @@ const assert = require('node:assert/strict');
 const fs = require('fs');
 const os = require('os');
 const path = require('path');
-const { buildScriptArgs, scriptSupportsBackfill, resolveSetupAllTools, clearUnboundEnvsEverywhere, NUKE_ENV_VARS, removeBinaryInstall, runBinaryClear, BINARY_INSTALL_PATHS } = require('../src/commands/setup');
+const { buildScriptArgs, scriptSupportsBackfill, resolveSetupAllTools, clearUnboundEnvsEverywhere, failureReason, NUKE_ENV_VARS, removeBinaryInstall, runBinaryClear, BINARY_INSTALL_PATHS } = require('../src/commands/setup');
 
 // shellEscape single-quotes every value, so a real key surfaces as
 // --api-key '<key>' at the head of the argv tail.
@@ -451,3 +451,170 @@ test('runBinaryClear: clear exits non-zero — fail-open (ran:true, ok:false), n
     fs.rmSync(tmp, { recursive: true, force: true });
   }
 });
+
+// The python clears open AND close with a `====` banner, so the old
+// `.split('\n')[0]` reason rendered `Codex (gateway) (==================)` —
+// which is what hid the real MDM-clear bug. Prefer the script's own error line.
+test('failureReason: skips the ==== banner and reports the script error line', () => {
+  const err = new Error([
+    '============================================================',
+    'Codex - Clearing MDM Setup',
+    '============================================================',
+    '',
+    'Clearing environment variables...',
+    'Failed to clear openai_base_url in codex config for alice',
+    '',
+    'Clear Complete!',
+  ].join('\n'));
+  err.exitCode = 1;
+  assert.equal(failureReason(err), 'Failed to clear openai_base_url in codex config for alice');
+});
+
+test('failureReason: picks the LAST flagged line when a script reports several', () => {
+  const err = new Error('Failed to clear API_KEY for 1 user(s)\nFailed to clear BASE_URL for 1 user(s)');
+  assert.equal(failureReason(err), 'Failed to clear BASE_URL for 1 user(s)');
+});
+
+test('failureReason: recognises the scripts’ ❌ prefix', () => {
+  const err = new Error('====\nSetup\n====\n❌ No user home directories found');
+  err.exitCode = 1;
+  assert.equal(failureReason(err), '❌ No user home directories found');
+});
+
+// Banner-only output with no error line must not surface "Clear Complete!" as
+// the reason — the exit code is the honest answer there.
+test('failureReason: no flagged line falls back to the exit code, not the banner', () => {
+  const err = new Error('============\nCodex - Clearing MDM Setup\n============\n\nClear Complete!');
+  err.exitCode = 2;
+  assert.equal(failureReason(err), 'exit code 2');
+});
+
+// Download failures carry no exitCode; their message is already the reason.
+test('failureReason: download failure with no exitCode keeps its own message', () => {
+  const err = new Error('Download failed: 404 Not Found');
+  err.downloadFailed = true;
+  assert.equal(failureReason(err), 'Download failed: 404 Not Found');
+});
+
+test('failureReason: empty message never returns undefined', () => {
+  assert.equal(failureReason(new Error('')), 'failed');
+  assert.equal(failureReason({}), 'failed');
+});
+
+test('failureReason: long error lines are truncated to 80 chars', () => {
+  const err = new Error('Failed to ' + 'x'.repeat(200));
+  assert.equal(failureReason(err).length, 80);
+});
+
+// A python crash prints a traceback whose LAST line is the real cause. Falling
+// back to `exit code 1` there would be less useful than the old first-line rule.
+test('failureReason: python traceback reports the exception, not the header', () => {
+  const err = new Error([
+    'Traceback (most recent call last):',
+    '  File "/tmp/setup.py", line 704, in clear_setup',
+    '    remove_codex_config_base_url_for_user(username, home_dir)',
+    "NameError: name 'home_dir' is not defined",
+  ].join('\n'));
+  err.exitCode = 1;
+  assert.equal(failureReason(err), "NameError: name 'home_dir' is not defined");
+});
+
+// exitCode 0 is falsy — must not fall through to the banner-first branch.
+// Module-qualified exception names have dots, so a name-shaped regex misses
+// them. The traceback tail is taken instead, whatever the exception is called.
+test('failureReason: dotted exception name in a traceback is reported', () => {
+  const err = new Error([
+    'Traceback (most recent call last):',
+    '  File "/tmp/setup.py", line 800, in clear_managed_hooks',
+    '    settings = json.load(f)',
+    'json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)',
+  ].join('\n'));
+  err.exitCode = 1;
+  assert.equal(failureReason(err), 'json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)');
+});
+
+// Not every python exception name contains "Error"/"Exception".
+test('failureReason: traceback tail works for a non-Error exception name', () => {
+  const err = new Error('Traceback (most recent call last):\n  File "x.py", line 1\nsocket.timeout: timed out');
+  err.exitCode = 1;
+  assert.equal(failureReason(err), 'socket.timeout: timed out');
+});
+
+// A bare exception line captured without its traceback header still flags.
+test('failureReason: dotted exception line without a traceback header flags', () => {
+  const err = new Error('====\nCodex - Clearing MDM Setup\n====\nsubprocess.CalledProcessError: returned non-zero exit status 1');
+  err.exitCode = 1;
+  assert.equal(failureReason(err), 'subprocess.CalledProcessError: returned non-zero exit status 1');
+});
+
+// Banner lines after the traceback must not be mistaken for the cause.
+test('failureReason: banner trailing a traceback is skipped', () => {
+  const err = new Error('Traceback (most recent call last):\nValueError: bad\n============');
+  err.exitCode = 1;
+  assert.equal(failureReason(err), 'ValueError: bad');
+});
+
+// runPythonScript merges stdout+stderr by arrival order and python block-buffers
+// stdout when piped, so a crash's progress lines can land AFTER the traceback.
+// Picking the tail would report "Clear Complete!" as the failure reason.
+test('failureReason: stdout flushed after the traceback does not displace the exception', () => {
+  const err = new Error([
+    'Traceback (most recent call last):',
+    '  File "/tmp/setup.py", line 9, in clear_setup',
+    'ValueError: the real cause',
+    '============================================================',
+    'Clear Complete!',
+  ].join('\n'));
+  err.exitCode = 1;
+  assert.equal(failureReason(err), 'ValueError: the real cause');
+});
+
+// stderr is captured separately precisely so interleaving can't matter.
+test('failureReason: prefers the clean stderr stream over the merged output', () => {
+  const err = new Error('====\nClearing environment variables...\nTraceback (most recent call last):\nOSError: merged and interleaved\nClear Complete!');
+  err.stderr = 'Traceback (most recent call last):\n  File "x.py", line 1\nPermissionError: from stderr';
+  err.exitCode = 1;
+  assert.equal(failureReason(err), 'PermissionError: from stderr');
+});
+
+// Chained exceptions: the LAST traceback is the one that actually propagated.
+test('failureReason: chained exceptions report the final one', () => {
+  const err = new Error([
+    'Traceback (most recent call last):',
+    '  File "x.py", line 1',
+    'ValueError: the first one',
+    'During handling of the above exception, another exception occurred:',
+    'Traceback (most recent call last):',
+    '  File "x.py", line 5',
+    'RuntimeError: the one that propagated',
+  ].join('\n'));
+  err.exitCode = 1;
+  assert.equal(failureReason(err), 'RuntimeError: the one that propagated');
+});
+
+// Indented frame lines are never the reason.
+test('failureReason: never reports an indented traceback frame line', () => {
+  const err = new Error('Traceback (most recent call last):\n  File "x.py", line 1, in f\n    boom()\nKeyError: \'gateway_url\'');
+  err.exitCode = 1;
+  assert.equal(failureReason(err), "KeyError: 'gateway_url'");
+});
+
+test('failureReason: exitCode 0 is reported, not treated as absent', () => {
+  assert.equal(failureReason({ message: '====\nCodex\n====', exitCode: 0 }), 'exit code 0');
+});
+
+// Truncation slices code points, so an emoji at the boundary stays intact.
+test('failureReason: truncation never splits an astral character', () => {
+  const reason = failureReason(new Error('Failed ' + '🔥'.repeat(100)));
+  assert.equal(Array.from(reason).length, 80);
+  assert.ok(!reason.includes('\ufffd'), reason);
+  assert.ok(/🔥$/.test(reason), reason);
+});
+
+// Runs inside nuke's per-tool catch — throwing here would abort the remaining
+// tools instead of clearing them, so a non-Error throw must not blow up.
+test('failureReason: non-Error throw (null/undefined) never throws', () => {
+  assert.equal(failureReason(null), 'failed');
+  assert.equal(failureReason(undefined), 'failed');
+  assert.equal(failureReason('a string'), 'failed');
+});