tachibot-mcp 2.19.3 → 2.21.1

package/CHANGELOG.md

@@ -5,6 +5,54 @@ All notable changes to TachiBot MCP will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.21.1] - 2026-04-26
+
+### Changed
+- **OpenAI: gpt-5.4 → gpt-5.5** (released 2026-04-23). Agentic-focused, 1.1M context, omnimodal. Pricing $5/$30 per M tokens. `gpt-5.5-pro` ($30/$180) for premium tier. `gpt-5.4-mini` retained for `code`/`explain` tools (no `gpt-5.5-mini` released yet).
+- **Kimi: kimi-k2.5 → kimi-k2.6** (released 2026-04-20). 1T MoE, leads SWE-bench Pro for long-horizon coding. Pricing ~$0.74/$4.65 per M tokens. K2.5 retained as fallback.
+
+### Added
+- **Qwen3.6-Plus** (`qwen/qwen3.6-plus`) registered in `QWEN_MODELS.PLUS_3_6`. New April 2026 general-purpose flagship at $0.325/$1.95 per M. Not yet wired as default — `qwen3-coder-next` (coder) and `qwen3-235b-thinking-2507` (reason) remain primary; awaiting `qwen3.6-coder` variant.
+- Display names + pricing for `gpt-5.5`, `gpt-5.5-pro`, `kimi-k2.6`, `qwen3.6-plus`, `qwen3-235b-a22b-thinking-2507`.
+- Auto-fallback: `kimi-k2.6` → `kimi-k2.5` on quota errors.
+
+### Notes
+- Grok 5 not released (Q2 2026 expected). Keeping `grok-4.20-0309-reasoning`.
+- Gemini 3.5 in preview, GA expected at Google I/O May 2026. Keeping `gemini-3.1-pro-preview`.
+- Verified all model IDs against live OpenAI `/v1/models` and OpenRouter `/v1/models` endpoints before release.
+
+## [2.21.0] - 2026-04-13
+
+### Added
+- **Auto-alias param names** — `z.preprocess()` hook in `safeAddTool` remaps `query` ↔ `problem` ↔ `prompt` ↔ `question` ↔ `topic` before Zod validation. LLMs that reach for the wrong synonym now succeed instead of hard-failing with `-32602 InvalidParams`.
+- **Zero per-tool changes** — single source of truth in `src/utils/param-aliases.ts`; every tool benefits automatically.
+- **11 unit tests** in `src/utils/__tests__/param-aliases.test.ts` covering directional aliasing, primary-wins precedence, and missing-key behavior.
+
+### Notes
+- Primary param value always wins when both primary and alias are provided.
+- Aliasing is transparent to tool implementations — the Zod schema sees the canonical key.
+
+## [2.20.0] - 2026-04-10
+
+### Changed
+- **Grok 4 → 4.20** — all defaults moved to flagship.
+  - `grok_reason` / `grok_search` → `grok-4.20-0309-reasoning` (low hallucination, 2M context)
+  - `grok_architect` → `grok-4.20-multi-agent-0309` (4–16 parallel agents)
+  - `grok_code` / `grok_debug` / `grok_brainstorm` → `grok-4.20-0309-non-reasoning` (fast turn-around)
+- **Smart timeout defaults bumped** — OpenAI 20→60s base, Grok max 90→120s.
+
+### Added
+- **AbortController on OpenAI** — 90s default, 180s for high-reasoning. No more hung calls.
+- **AbortController on Grok** — 60–180s based on model.
+- **`reasoning` param** on `callGrok` for multi-agent invocation; unified `GrokModel` enum.
+
+### Fixed
+- Stale `gpt-4-mini` → `gpt-5.4-mini` in architect + workflows.
+- Hardcoded `grok-4-0709` references across 6 scattered files (OpenRouter gateway, ANSI badges, model-router, tool-mapper).
+
+### Docs
+- Updated `docs/API_KEYS.md` and `docs/TOOL_PARAMETERS.md` for Grok 4.20.
+
 ## [2.19.3] - 2026-03-21
 
 ### Fixed

package/dist/evals/analysis/aggregator.js

@@ -0,0 +1,92 @@
+/**
+ * P9: Aggregator — turns raw RunRows into one CellStats per
+ * (benchmark, displayed-protocol, model_set) group.
+ *
+ * Displayed protocol:
+ *   - For single-model protocol families (`single-baseline`, `best-of-n`,
+ *     `self-consistency`) → `${protocol}[${model_set}]`. This makes
+ *     `single-baseline[openai]` distinguishable from `single-baseline[kimi]`
+ *     downstream (and lets `delta-table` rank baselines independently).
+ *   - For multi-model protocols (`parallel-council`, `sequential-pipeline`,
+ *     `adversarial-debate`) → just `${protocol}`. The lineup is implied.
+ *
+ * Skip semantics:
+ *   - Rows with `error !== null` OR `score === null` are excluded from
+ *     accuracy and cost-per-correct computations.
+ *   - errorRate is the fraction of *all* rows in the cell that had `error` set
+ *     (independent of score).
+ */
+import { bootstrapCi } from './bootstrap-ci.js';
+const SINGLE_MODEL_FAMILIES = new Set([
+    'single-baseline',
+    'best-of-n',
+    'self-consistency',
+]);
+function displayProtocol(row) {
+    if (SINGLE_MODEL_FAMILIES.has(row.protocol)) {
+        return `${row.protocol}[${row.model_set}]`;
+    }
+    return row.protocol;
+}
+export function aggregate(rows, opts = {}) {
+    const buckets = new Map();
+    for (const row of rows) {
+        const displayed = displayProtocol(row);
+        const key = `${row.benchmark}|${displayed}|${row.model_set}`;
+        let b = buckets.get(key);
+        if (!b) {
+            b = {
+                benchmark: row.benchmark,
+                displayedProtocol: displayed,
+                model_set: row.model_set,
+                scores: [],
+                costs: [],
+                latencies: [],
+                totalRows: 0,
+                errorRows: 0,
+            };
+            buckets.set(key, b);
+        }
+        b.totalRows += 1;
+        if (row.error !== null && row.error !== undefined && row.error !== '') {
+            b.errorRows += 1;
+        }
+        const valid = (row.error === null || row.error === undefined || row.error === '') &&
+            row.score !== null;
+        if (valid) {
+            b.scores.push(row.score);
+            b.costs.push(row.cost_usd ?? 0);
+            b.latencies.push(row.latency_ms ?? 0);
+        }
+    }
+    const out = [];
+    for (const b of buckets.values()) {
+        const ci = bootstrapCi(b.scores, { resamples: opts.bootstrapResamples ?? 1000, seed: 1 });
+        const totalCostUsd = b.costs.reduce((a, c) => a + c, 0);
+        const correctCount = b.scores.reduce((a, s) => a + (s === 1 ? 1 : 0), 0);
+        const avgLatencyMs = b.latencies.length === 0
+            ? 0
+            : b.latencies.reduce((a, c) => a + c, 0) / b.latencies.length;
+        const costPerCorrect = correctCount === 0 ? Infinity : totalCostUsd / correctCount;
+        const errorRate = b.totalRows === 0 ? 0 : b.errorRows / b.totalRows;
+        out.push({
+            benchmark: b.benchmark,
+            protocol: b.displayedProtocol,
+            model_set: b.model_set,
+            n: b.scores.length,
+            mean: ci.mean,
+            ciLow: ci.ciLow,
+            ciHigh: ci.ciHigh,
+            totalCostUsd,
+            avgLatencyMs,
+            correctCount,
+            costPerCorrect,
+            errorRate,
+        });
+    }
+    // Stable sort: benchmark asc, then protocol asc.
+    out.sort((a, b) => a.benchmark === b.benchmark
+        ? a.protocol.localeCompare(b.protocol)
+        : a.benchmark.localeCompare(b.benchmark));
+    return out;
+}

package/dist/evals/analysis/bootstrap-ci.js

@@ -0,0 +1,58 @@
+/**
+ * P9: Bootstrap confidence interval for binary scores.
+ *
+ * Standard percentile-of-resamples bootstrap. Input is an array of 0/1
+ * (correct/incorrect). Output is the observed mean and the alpha/2 and
+ * 1-alpha/2 quantiles of the resampled-mean distribution.
+ *
+ * Determinism: a `seed` parameter feeds the same mulberry32 PRNG used by
+ * `evals/datasets/sample.ts`, so the same seed → identical CI bounds across
+ * Node versions.
+ *
+ * Edge cases:
+ *   scores.length === 0 → all NaN, n=0
+ *   scores.length === 1 → CI collapses to [score, score]
+ */
+import { mulberry32 } from '../datasets/sample.js';
+export function bootstrapCi(scores, opts = {}) {
+    const n = scores.length;
+    if (n === 0) {
+        return { mean: NaN, ciLow: NaN, ciHigh: NaN, n: 0 };
+    }
+    const mean = scores.reduce((a, b) => a + b, 0) / n;
+    if (n === 1) {
+        const v = scores[0];
+        return { mean: v, ciLow: v, ciHigh: v, n };
+    }
+    const resamples = Math.max(1, opts.resamples ?? 1000);
+    const alpha = opts.alpha ?? 0.05;
+    const seed = opts.seed ?? 1;
+    const rand = mulberry32(seed);
+    const means = new Array(resamples);
+    for (let r = 0; r < resamples; r++) {
+        let sum = 0;
+        for (let i = 0; i < n; i++) {
+            const idx = Math.floor(rand() * n);
+            sum += scores[idx];
+        }
+        means[r] = sum / n;
+    }
+    means.sort((a, b) => a - b);
+    const ciLow = quantile(means, alpha / 2);
+    const ciHigh = quantile(means, 1 - alpha / 2);
+    return { mean, ciLow, ciHigh, n };
+}
+/** Linear-interpolated quantile of a pre-sorted array. */
+function quantile(sorted, q) {
+    if (sorted.length === 0)
+        return NaN;
+    if (sorted.length === 1)
+        return sorted[0];
+    const pos = q * (sorted.length - 1);
+    const lo = Math.floor(pos);
+    const hi = Math.ceil(pos);
+    if (lo === hi)
+        return sorted[lo];
+    const frac = pos - lo;
+    return sorted[lo] + frac * (sorted[hi] - sorted[lo]);
+}

package/dist/evals/analysis/csv-reader.js

@@ -0,0 +1,137 @@
+/**
+ * P9: CSV reader — round-trips files written by `evals/writers/csv-writer.ts`.
+ *
+ * Implements RFC 4180 quoting:
+ *   - Fields may be wrapped in double-quotes.
+ *   - Inside a quoted field, `""` represents a literal `"`.
+ *   - Quoted fields may contain commas and newlines verbatim.
+ *
+ * Also handles the schema-specific coercion: numeric fields revert from
+ * strings to numbers, nullable fields render an empty CSV cell as null.
+ */
+import { promises as fs } from 'node:fs';
+import { CSV_COLUMNS } from '../writers/csv-writer.js';
+const NUMERIC_FIELDS = new Set([
+    'k_reps',
+    'run_idx',
+    'tokens_in',
+    'tokens_out',
+    'cost_usd',
+    'latency_ms',
+]);
+/** Score / ci_low / ci_high are number-or-null. */
+const NULLABLE_NUMERIC_FIELDS = new Set([
+    'score',
+    'ci_low',
+    'ci_high',
+]);
+/** Strings that may be null when empty. */
+const NULLABLE_STRING_FIELDS = new Set([
+    'judge_id',
+    'error',
+]);
+export async function readCsv(filepath) {
+    const text = await fs.readFile(filepath, 'utf8');
+    const records = parseCsv(text);
+    if (records.length === 0)
+        return [];
+    const header = records[0];
+    // Sanity: header must match expected columns in order.
+    for (let i = 0; i < CSV_COLUMNS.length; i++) {
+        if (header[i] !== CSV_COLUMNS[i]) {
+            throw new Error(`csv-reader: unexpected header at column ${i}: got "${header[i]}", expected "${CSV_COLUMNS[i]}"`);
+        }
+    }
+    const rows = [];
+    for (let r = 1; r < records.length; r++) {
+        const fields = records[r];
+        if (fields.length === 1 && fields[0] === '')
+            continue; // blank trailing line
+        const row = {};
+        for (let c = 0; c < CSV_COLUMNS.length; c++) {
+            const col = CSV_COLUMNS[c];
+            const raw = fields[c] ?? '';
+            row[col] = coerce(col, raw);
+        }
+        rows.push(row);
+    }
+    return rows;
+}
+function coerce(col, raw) {
+    if (NUMERIC_FIELDS.has(col)) {
+        return raw === '' ? 0 : Number(raw);
+    }
+    if (NULLABLE_NUMERIC_FIELDS.has(col)) {
+        return raw === '' ? null : Number(raw);
+    }
+    if (NULLABLE_STRING_FIELDS.has(col)) {
+        return raw === '' ? null : raw;
+    }
+    // Plain string fields.
+    return raw;
+}
+/**
+ * RFC 4180 streaming-state-machine parser. Returns an array of records,
+ * each record being an array of field strings.
+ */
+function parseCsv(text) {
+    const records = [];
+    let field = '';
+    let record = [];
+    let inQuotes = false;
+    let i = 0;
+    const len = text.length;
+    while (i < len) {
+        const ch = text[i];
+        if (inQuotes) {
+            if (ch === '"') {
+                if (i + 1 < len && text[i + 1] === '"') {
+                    // Escaped quote.
+                    field += '"';
+                    i += 2;
+                    continue;
+                }
+                // End of quoted field.
+                inQuotes = false;
+                i++;
+                continue;
+            }
+            field += ch;
+            i++;
+            continue;
+        }
+        if (ch === '"') {
+            inQuotes = true;
+            i++;
+            continue;
+        }
+        if (ch === ',') {
+            record.push(field);
+            field = '';
+            i++;
+            continue;
+        }
+        if (ch === '\n' || ch === '\r') {
+            // End of record. Handle CRLF.
+            record.push(field);
+            records.push(record);
+            record = [];
+            field = '';
+            if (ch === '\r' && i + 1 < len && text[i + 1] === '\n') {
+                i += 2;
+            }
+            else {
+                i++;
+            }
+            continue;
+        }
+        field += ch;
+        i++;
+    }
+    // Flush trailing field/record (file may not end with newline).
+    if (field.length > 0 || record.length > 0) {
+        record.push(field);
+        records.push(record);
+    }
+    return records;
+}

package/dist/evals/analysis/delta-table.js

@@ -0,0 +1,60 @@
+/**
+ * P9: Delta table — for each benchmark, picks the best single-baseline cell
+ * and the best protocol-overall cell, then computes Δaccuracy and
+ * Δcost-per-correct between them.
+ *
+ * Note: "best protocol overall" is allowed to BE a single-baseline. In that
+ * case the delta is 0 — the multi-model machinery didn't help on that bench.
+ * That's a legitimate finding and should be reported, not hidden.
+ *
+ * CI overlap: simple interval-overlap test. Two intervals [a, b] and [c, d]
+ * overlap iff a <= d AND c <= b. When CIs overlap, the absolute Δ is not
+ * statistically significant at the chosen alpha (95% by default).
+ */
+function intervalsOverlap(a, b, c, d) {
+    return a <= d && c <= b;
+}
+export function buildDeltaTable(stats) {
+    // Group by benchmark.
+    const byBench = new Map();
+    for (const s of stats) {
+        const bucket = byBench.get(s.benchmark);
+        if (bucket)
+            bucket.push(s);
+        else
+            byBench.set(s.benchmark, [s]);
+    }
+    const out = [];
+    for (const [benchmark, cells] of byBench) {
+        const baselines = cells.filter((c) => c.protocol.startsWith('single-baseline'));
+        if (baselines.length === 0)
+            continue; // Nothing to compare against.
+        const bestBaseline = baselines.reduce((a, b) => (b.mean > a.mean ? b : a));
+        const bestOverall = cells.reduce((a, b) => (b.mean > a.mean ? b : a));
+        out.push({
+            benchmark,
+            bestSingleBaseline: {
+                protocol: bestBaseline.protocol,
+                model: bestBaseline.model_set,
+                accuracy: bestBaseline.mean,
+                ciLow: bestBaseline.ciLow,
+                ciHigh: bestBaseline.ciHigh,
+                costPerCorrect: bestBaseline.costPerCorrect,
+            },
+            bestProtocol: {
+                protocol: bestOverall.protocol,
+                model_set: bestOverall.model_set,
+                accuracy: bestOverall.mean,
+                ciLow: bestOverall.ciLow,
+                ciHigh: bestOverall.ciHigh,
+                costPerCorrect: bestOverall.costPerCorrect,
+            },
+            deltaAccuracy: bestOverall.mean - bestBaseline.mean,
+            deltaCostPerCorrect: bestOverall.costPerCorrect - bestBaseline.costPerCorrect,
+            ciOverlap: intervalsOverlap(bestBaseline.ciLow, bestBaseline.ciHigh, bestOverall.ciLow, bestOverall.ciHigh),
+        });
+    }
+    // Stable ordering: benchmark name asc.
+    out.sort((a, b) => a.benchmark.localeCompare(b.benchmark));
+    return out;
+}