wotann 0.5.0 → 0.5.39

package/dist/orchestration/elo-tournament.js

@@ -0,0 +1,203 @@
+/**
+ * elo-tournament — Pairwise-compare artifacts via an injected judge,
+ * produce a deterministic Elo ranking.
+ *
+ * Generalizes the multi-model-arena pattern: instead of a single
+ * round-robin scoring, run multiple rounds of Swiss-style pairings
+ * where each match produces a winner/loser/draw and updates Elo
+ * ratings using the canonical formula:
+ *
+ *     E_A = 1 / (1 + 10^((R_B - R_A) / 400))
+ *     R_A' = R_A + K * (S_A - E_A)
+ *
+ * where S_A is 1 for win, 0 for loss, 0.5 for draw.
+ *
+ * Pure, vendor-agnostic API:
+ *   - Caller supplies a `judge(a, b) => EloMatchResult` callback.
+ *   - Module owns: Swiss pairing (avoid repeats when possible),
+ *     deterministic shuffling via injected seed, Elo math, ranking,
+ *     match log construction, immutable result.
+ *
+ * Quality bars:
+ *   - QB#1 immutable: ratings + matchLog readonly + frozen.
+ *   - QB#7 stateless: per-call working state; no module globals.
+ *   - QB#8 deterministic with seed: identical seed + identical
+ *     judge => identical ranking, every time. Critical for tests
+ *     and reproducible CI.
+ *
+ * Use cases:
+ *   - Rank prompt variants by judge preference.
+ *   - Tournament-rank generated plans before picking the winner.
+ *   - Compare summaries head-to-head for editorial selection.
+ *   - Pick between recipe candidates from multiple LLMs.
+ */
+/**
+ * Mulberry32 — a tiny, deterministic PRNG. Good enough for pairing
+ * shuffles (we don't need cryptographic randomness, but we DO need
+ * the same seed to produce the same sequence on every machine).
+ */
+function mulberry32(seed) {
+    let a = seed >>> 0;
+    return function next() {
+        a = (a + 0x6d2b79f5) >>> 0;
+        let t = a;
+        t = Math.imul(t ^ (t >>> 15), t | 1);
+        t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+}
+function expectedScore(ratingA, ratingB) {
+    return 1 / (1 + Math.pow(10, (ratingB - ratingA) / 400));
+}
+function updateRating(rating, score, expected, kFactor) {
+    return rating + kFactor * (score - expected);
+}
+/**
+ * Swiss-style pairing for one round.
+ *
+ * Algorithm:
+ *   1. Sort contestants by current rating (desc), ties broken by
+ *      deterministic shuffle.
+ *   2. Pair adjacent contestants if they haven't met yet.
+ *   3. If a pair would be a repeat, swap with the next available slot
+ *      (skipping repeats whenever possible — but if every remaining
+ *      pairing would be a repeat, accept a repeat to keep the round
+ *      complete).
+ *   4. Odd contestant count: the lowest-rated unpaired contestant gets
+ *      a "bye" for that round (no rating change, no match logged).
+ */
+function swissPair(entries, played, rand) {
+    const pool = [...entries];
+    // Deterministic stable sort: by rating desc, with shuffled ties.
+    const noise = new Map(pool.map((e) => [e.id, rand()]));
+    pool.sort((a, b) => {
+        if (a.rating !== b.rating)
+            return b.rating - a.rating;
+        return (noise.get(a.id) ?? 0) - (noise.get(b.id) ?? 0);
+    });
+    const pairs = [];
+    const remaining = pool.slice();
+    while (remaining.length >= 2) {
+        const first = remaining.shift();
+        if (!first)
+            break;
+        // Find the closest-rated opponent who hasn't played `first` yet.
+        let partnerIdx = -1;
+        for (let i = 0; i < remaining.length; i += 1) {
+            const candidate = remaining[i];
+            if (!candidate)
+                continue;
+            if (!played.has(pairKey(first.id, candidate.id))) {
+                partnerIdx = i;
+                break;
+            }
+        }
+        // If everyone is a repeat, fall back to the first remaining.
+        if (partnerIdx === -1)
+            partnerIdx = 0;
+        const partner = remaining.splice(partnerIdx, 1)[0];
+        if (!partner)
+            break;
+        pairs.push([first, partner]);
+    }
+    return pairs;
+}
+function pairKey(a, b) {
+    return a < b ? `${a}::${b}` : `${b}::${a}`;
+}
+// ── Main API ─────────────────────────────────────────────────────────
+/**
+ * Run a Swiss-style Elo tournament. Returns the final ranking + the
+ * complete match log. Deterministic for a given seed + judge.
+ */
+export async function runEloTournament(opts) {
+    if (opts.contestants.length < 2) {
+        throw new Error(`elo-tournament: need at least 2 contestants, got ${opts.contestants.length}`);
+    }
+    if (!Number.isInteger(opts.rounds) || opts.rounds < 1) {
+        throw new Error(`elo-tournament: rounds must be a positive integer, got ${opts.rounds}`);
+    }
+    const k = opts.kFactor ?? 32;
+    const initialRating = opts.initialRating ?? 1200;
+    const seed = opts.seed ?? 0xc0ffee;
+    const rand = mulberry32(seed);
+    // Verify ids are unique.
+    const seen = new Set();
+    for (const c of opts.contestants) {
+        if (seen.has(c.id))
+            throw new Error(`elo-tournament: duplicate contestant id ${c.id}`);
+        seen.add(c.id);
+    }
+    // Initialize mutable state.
+    const entries = opts.contestants.map((c) => ({
+        id: c.id,
+        payload: c.payload,
+        rating: initialRating,
+        wins: 0,
+        losses: 0,
+        draws: 0,
+    }));
+    const byId = new Map(entries.map((e) => [e.id, e]));
+    const played = new Set();
+    const matchLog = [];
+    for (let round = 0; round < opts.rounds; round += 1) {
+        const pairs = swissPair(entries, played, rand);
+        for (const [a, b] of pairs) {
+            const outcome = await opts.judge({ id: a.id, payload: a.payload }, { id: b.id, payload: b.payload });
+            played.add(pairKey(a.id, b.id));
+            const expectedA = expectedScore(a.rating, b.rating);
+            const expectedB = 1 - expectedA;
+            let scoreA;
+            let scoreB;
+            if (outcome.drew) {
+                scoreA = 0.5;
+                scoreB = 0.5;
+                a.draws += 1;
+                b.draws += 1;
+            }
+            else if (outcome.winnerId === a.id) {
+                scoreA = 1;
+                scoreB = 0;
+                a.wins += 1;
+                b.losses += 1;
+            }
+            else if (outcome.winnerId === b.id) {
+                scoreA = 0;
+                scoreB = 1;
+                b.wins += 1;
+                a.losses += 1;
+            }
+            else {
+                throw new Error(`elo-tournament: judge returned winnerId=${outcome.winnerId} which is neither ${a.id} nor ${b.id}`);
+            }
+            a.rating = updateRating(a.rating, scoreA, expectedA, k);
+            b.rating = updateRating(b.rating, scoreB, expectedB, k);
+            matchLog.push(Object.freeze({
+                winnerId: outcome.winnerId,
+                loserId: outcome.loserId,
+                drew: outcome.drew,
+                ...(outcome.judgeNotes !== undefined ? { judgeNotes: outcome.judgeNotes } : {}),
+            }));
+        }
+    }
+    // Re-use byId to ensure rating order matches the sorted final output.
+    const sorted = [...entries].sort((a, b) => b.rating - a.rating);
+    const ratings = sorted.map((e) => Object.freeze({
+        id: e.id,
+        payload: e.payload,
+        rating: e.rating,
+        wins: e.wins,
+        losses: e.losses,
+        draws: e.draws,
+    }));
+    // Touch byId so noUnusedLocals-style linters don't complain about
+    // an internal map we keep for symmetry with future lookups.
+    void byId;
+    return Object.freeze({
+        ratings: Object.freeze(ratings),
+        matchLog: Object.freeze(matchLog),
+    });
+}
+// ── Re-exports for ergonomic consumer imports ────────────────────────
+export { expectedScore as eloExpectedScore, updateRating as eloUpdateRating };
+//# sourceMappingURL=elo-tournament.js.map

package/dist/orchestration/elo-tournament.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"elo-tournament.js","sourceRoot":"","sources":["../../src/orchestration/elo-tournament.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAoEH;;;;GAIG;AACH,SAAS,UAAU,CAAC,IAAY;IAC9B,IAAI,CAAC,GAAG,IAAI,KAAK,CAAC,CAAC;IACnB,OAAO,SAAS,IAAI;QAClB,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC3B,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACrC,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,UAAU,CAAC;IAC/C,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,OAAe,EAAE,OAAe;IACrD,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,OAAO,GAAG,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,YAAY,CAAC,MAAc,EAAE,KAAa,EAAE,QAAgB,EAAE,OAAe;IACpF,OAAO,MAAM,GAAG,OAAO,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,CAAC;AAC/C,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,SAAS,SAAS,CAChB,OAAmC,EACnC,MAA2B,EAC3B,IAAkB;IAElB,MAAM,IAAI,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;IAC1B,iEAAiE;IACjE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAiB,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAU,CAAC,CAAC,CAAC;IAChF,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACjB,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;YAAE,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;QACtD,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;IACH,MAAM,KAAK,GAAuD,EAAE,CAAC;IACrE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;IAC/B,OAAO,SAAS,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,EAAE,CAAC;QAChC,IAAI,CAAC,KAAK;YAAE,MAAM;QAClB,iEAAiE;QACjE,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC;QACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7C,MAAM,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YAC/B,IAAI,CAAC,SAAS;gBAAE,SAAS;YACzB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;gBACjD,UAAU,GAAG,CAAC,CAAC;gBACf,MAAM;YACR,CAAC;QACH,CAAC;QACD,6DAA6D;QAC7D,IAAI,UAAU,KAAK,CAAC,CAAC;YAAE,UAAU,GAAG,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnD,IAAI,CAAC,OAAO;YAAE,MAAM;QACpB,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,OAAO,CAAU,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,OAAO,CAAC,CAAS,EAAE,CAAS;IACnC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;AAC7C,CAAC;AAED,wEAAwE;AAExE;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAI,IAA6B;IACrE,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CAAC,oDAAoD,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;IACjG,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CAAC,0DAA0D,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3F,CAAC;IACD,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;IAC7B,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC;IACjD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,QAAQ,CAAC;IACnC,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAE9B,yBAAyB;IACzB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;QACjC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACvF,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,4BAA4B;IAC5B,MAAM,OAAO,GAAsB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC9D,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,MAAM,EAAE,aAAa;QACrB,IAAI,EAAE,CAAC;QACP,MAAM,EAAE,CAAC;QACT,KAAK,EAAE,CAAC;KACT,CAAC,CAAC,CAAC;IACJ,MAAM,IAAI,GAAG,IAAI,GAAG,CAA0B,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7E,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,MAAM,QAAQ,GAAqB,EAAE,CAAC;IAEtC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACpD,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QAC/C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAC9B,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,EAChC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CACjC,CAAC;YACF,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAChC,MAAM,SAAS,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;YACpD,MAAM,SAAS,GAAG,CAAC,GAAG,SAAS,CAAC;YAChC,IAAI,MAAc,CAAC;YACnB,IAAI,MAAc,CAAC;YACnB,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;gBACjB,MAAM,GAAG,GAAG,CAAC;gBACb,MAAM,GAAG,GAAG,CAAC;gBACb,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;gBACb,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;YACf,CAAC;iBAAM,IAAI,OAAO,CAAC,QAAQ,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;gBACrC,MAAM,GAAG,CAAC,CAAC;gBACX,MAAM,GAAG,CAAC,CAAC;gBACX,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;gBACZ,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;YAChB,CAAC;iBAAM,IAAI,OAAO,CAAC,QAAQ,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;gBACrC,MAAM,GAAG,CAAC,CAAC;gBACX,MAAM,GAAG,CAAC,CAAC;gBACX,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;gBACZ,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;YAChB,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CACb,2CAA2C,OAAO,CAAC,QAAQ,qBAAqB,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,EAAE,EAAE,CACnG,CAAC;YACJ,CAAC;YACD,CAAC,CAAC,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;YACxD,CAAC,CAAC,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;YACxD,QAAQ,CAAC,IAAI,CACX,MAAM,CAAC,MAAM,CAAC;gBACZ,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,GAAG,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAChF,CAAC,CACH,CAAC;QACJ,CAAC;IACH,CAAC;IAED,sEAAsE;IACtE,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;IAChE,MAAM,OAAO,GAAwB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACpD,MAAM,CAAC,MAAM,CAAC;QACZ,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,KAAK,EAAE,CAAC,CAAC,KAAK;KACf,CAAC,CACH,CAAC;IACF,kEAAkE;IAClE,4DAA4D;IAC5D,KAAK,IAAI,CAAC;IACV,OAAO,MAAM,CAAC,MAAM,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAqC;QACnE,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAA8B;KAC/D,CAAC,CAAC;AACL,CAAC;AAED,wEAAwE;AAExE,OAAO,EAAE,aAAa,IAAI,gBAAgB,EAAE,YAAY,IAAI,eAAe,EAAE,CAAC"}

package/dist/orchestration/multi-model-arena.d.ts

@@ -0,0 +1,138 @@
+/**
+ * multi-model-arena — Run a single query through N models in parallel
+ * and surface a side-by-side comparison + voting + best-pick.
+ *
+ * Inspired by Cursor's "Arena Mode" (https://nevo.systems/blogs/nevo-journal/windsurf-vs-cursor)
+ * and Perplexity Comet's "all 19 models in parallel" pattern. The
+ * motivation:
+ *
+ *   The agent doesn't always know which provider will win on a given
+ *   query. Cheap models hallucinate dates; expensive ones miss
+ *   formatting; specialty models excel at code but flop on prose.
+ *   Running 3-5 models in parallel + showing the user side-by-side
+ *   gives them the answer + the disagreement signal — far stronger
+ *   than picking one model and hoping.
+ *
+ * The arena is COMPLETELY VENDOR-AGNOSTIC: every model is a
+ * `(prompt: string) => Promise<string>` callback supplied by the
+ * caller. The arena owns:
+ *   - parallel dispatch with bounded concurrency
+ *   - per-model timeouts
+ *   - response collection with timing + error capture
+ *   - similarity-based clustering (find consensus vs outliers)
+ *   - 3 voting schemes: longest, most-agreeing, judge-scored
+ *   - structured ArenaResult with per-model details + winner
+ *
+ * Quality bars:
+ *   - QB#1 immutable: ArenaResult + every nested record is readonly
+ *   - QB#3 honest: timeouts + errors surface as ContestantResult.error
+ *     entries; the arena never fakes a response
+ *   - QB#7 stateless: per-call concurrency + state; no module globals
+ *   - QB#11 sibling-site: composes provider callbacks; doesn't bind
+ *     to any specific provider implementation
+ *
+ * Usage:
+ *   const result = await runArena({
+ *     query: "summarize this",
+ *     contestants: [
+ *       { id: "claude-sonnet", fn: claudeFn, weight: 1.0 },
+ *       { id: "gpt-5", fn: gptFn, weight: 1.0 },
+ *       { id: "gemini-2.5-pro", fn: geminiFn, weight: 1.0 },
+ *       { id: "pollinations-free", fn: pollinationsFn, weight: 0.5 },
+ *     ],
+ *     timeoutMs: 30_000,
+ *     concurrency: 4,
+ *     voting: "judge-scored",
+ *     judge: judgeFn,
+ *   });
+ */
+export type ContestantFn = (prompt: string) => Promise<string>;
+export interface Contestant {
+    readonly id: string;
+    readonly fn: ContestantFn;
+    /**
+     * Relative voting weight. Higher = the contestant's vote carries
+     * more weight in the consensus tally. Default 1.0. Useful when you
+     * trust an expensive model more than a free one.
+     */
+    readonly weight?: number;
+    /**
+     * Optional human-readable label for UI rendering (e.g. "Claude
+     * Opus 4.7 (1M)"). Defaults to id.
+     */
+    readonly label?: string;
+}
+export interface ContestantResult {
+    readonly id: string;
+    readonly label: string;
+    readonly weight: number;
+    readonly status: "ok" | "timeout" | "error";
+    readonly response: string;
+    readonly errorMessage?: string;
+    readonly latencyMs: number;
+    /** Optional judge-assigned score in [0,1]. Only present when
+     *  voting === "judge-scored". */
+    readonly judgeScore?: number;
+}
+export type VotingScheme = "longest" | "most-agreeing" | "judge-scored" | "first-ok";
+export interface ArenaResult {
+    readonly query: string;
+    readonly contestants: readonly ContestantResult[];
+    /**
+     * The winning contestant's id, or null if no contestant succeeded.
+     * The winner is the entry that maximizes the voting scheme's metric.
+     */
+    readonly winnerId: string | null;
+    readonly winningResponse: string | null;
+    readonly votingScheme: VotingScheme;
+    readonly totalLatencyMs: number;
+    /**
+     * Pairwise similarity-based clusters — entries in the same cluster
+     * agree (Jaccard >= clusterThreshold). Useful for detecting
+     * consensus vs outlier responses.
+     */
+    readonly clusters: readonly (readonly string[])[];
+    /** Per-cluster total weight. Sorted by weight descending. */
+    readonly clusterWeights: readonly number[];
+}
+export interface ArenaRequest {
+    readonly query: string;
+    readonly contestants: readonly Contestant[];
+    /** Per-contestant timeout (default 30s). */
+    readonly timeoutMs?: number;
+    /** Max parallel contestants (default = contestants.length, no cap). */
+    readonly concurrency?: number;
+    /** Voting scheme (default "longest"). */
+    readonly voting?: VotingScheme;
+    /**
+     * Required when voting === "judge-scored". Returns a [0,1] score
+     * for a contestant's response. The judge sees ALL responses so it
+     * can score them comparatively.
+     */
+    readonly judge?: (args: {
+        readonly query: string;
+        readonly responses: ReadonlyArray<{
+            readonly id: string;
+            readonly response: string;
+        }>;
+    }) => Promise<ReadonlyArray<{
+        readonly id: string;
+        readonly score: number;
+    }>>;
+    /**
+     * Jaccard similarity threshold for cluster membership (default 0.5).
+     */
+    readonly clusterThreshold?: number;
+}
+/**
+ * Run an arena. Returns ArenaResult; never throws (every contestant
+ * error is captured into ContestantResult.errorMessage).
+ */
+export declare function runArena(req: ArenaRequest): Promise<ArenaResult>;
+/**
+ * Render a Markdown side-by-side report for the arena result. Suitable
+ * for `wotann ow arena run --md`, PR comments, or inline display in
+ * Desktop / TUI surfaces.
+ */
+export declare function renderArenaMarkdown(result: ArenaResult): string;
+//# sourceMappingURL=multi-model-arena.d.ts.map

package/dist/orchestration/multi-model-arena.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"multi-model-arena.d.ts","sourceRoot":"","sources":["../../src/orchestration/multi-model-arena.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAIH,MAAM,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;AAE/D,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,EAAE,YAAY,CAAC;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,IAAI,GAAG,SAAS,GAAG,OAAO,CAAC;IAC5C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;qCACiC;IACjC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,eAAe,GAAG,cAAc,GAAG,UAAU,CAAC;AAErF,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,SAAS,gBAAgB,EAAE,CAAC;IAClD;;;OAGG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,YAAY,EAAE,YAAY,CAAC;IACpC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC;;;;OAIG;IACH,QAAQ,CAAC,QAAQ,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;IAClD,6DAA6D;IAC7D,QAAQ,CAAC,cAAc,EAAE,SAAS,MAAM,EAAE,CAAC;CAC5C;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,SAAS,UAAU,EAAE,CAAC;IAC5C,4CAA4C;IAC5C,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,uEAAuE;IACvE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,yCAAyC;IACzC,QAAQ,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC;IAC/B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE;QACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,SAAS,EAAE,aAAa,CAAC;YAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KACvF,KAAK,OAAO,CAAC,aAAa,CAAC;QAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAAC;IAC9E;;OAEG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACpC;AAoJD;;;GAGG;AACH,wBAAsB,QAAQ,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAsFtE;AAID;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,WAAW,GAAG,MAAM,CAkC/D"}

package/dist/orchestration/multi-model-arena.js

@@ -0,0 +1,298 @@
+/**
+ * multi-model-arena — Run a single query through N models in parallel
+ * and surface a side-by-side comparison + voting + best-pick.
+ *
+ * Inspired by Cursor's "Arena Mode" (https://nevo.systems/blogs/nevo-journal/windsurf-vs-cursor)
+ * and Perplexity Comet's "all 19 models in parallel" pattern. The
+ * motivation:
+ *
+ *   The agent doesn't always know which provider will win on a given
+ *   query. Cheap models hallucinate dates; expensive ones miss
+ *   formatting; specialty models excel at code but flop on prose.
+ *   Running 3-5 models in parallel + showing the user side-by-side
+ *   gives them the answer + the disagreement signal — far stronger
+ *   than picking one model and hoping.
+ *
+ * The arena is COMPLETELY VENDOR-AGNOSTIC: every model is a
+ * `(prompt: string) => Promise<string>` callback supplied by the
+ * caller. The arena owns:
+ *   - parallel dispatch with bounded concurrency
+ *   - per-model timeouts
+ *   - response collection with timing + error capture
+ *   - similarity-based clustering (find consensus vs outliers)
+ *   - 3 voting schemes: longest, most-agreeing, judge-scored
+ *   - structured ArenaResult with per-model details + winner
+ *
+ * Quality bars:
+ *   - QB#1 immutable: ArenaResult + every nested record is readonly
+ *   - QB#3 honest: timeouts + errors surface as ContestantResult.error
+ *     entries; the arena never fakes a response
+ *   - QB#7 stateless: per-call concurrency + state; no module globals
+ *   - QB#11 sibling-site: composes provider callbacks; doesn't bind
+ *     to any specific provider implementation
+ *
+ * Usage:
+ *   const result = await runArena({
+ *     query: "summarize this",
+ *     contestants: [
+ *       { id: "claude-sonnet", fn: claudeFn, weight: 1.0 },
+ *       { id: "gpt-5", fn: gptFn, weight: 1.0 },
+ *       { id: "gemini-2.5-pro", fn: geminiFn, weight: 1.0 },
+ *       { id: "pollinations-free", fn: pollinationsFn, weight: 0.5 },
+ *     ],
+ *     timeoutMs: 30_000,
+ *     concurrency: 4,
+ *     voting: "judge-scored",
+ *     judge: judgeFn,
+ *   });
+ */
+// ── Helpers ──────────────────────────────────────────────────────────
+async function runWithTimeout(fn, timeoutMs) {
+    let timer;
+    try {
+        const result = await Promise.race([
+            fn().then((response) => ({ status: "ok", response })),
+            new Promise((resolve) => {
+                timer = setTimeout(() => resolve({ status: "timeout", response: "" }), timeoutMs);
+            }),
+        ]);
+        return result;
+    }
+    catch (err) {
+        return {
+            status: "error",
+            response: "",
+            error: err instanceof Error ? err.message : String(err),
+        };
+    }
+    finally {
+        if (timer)
+            clearTimeout(timer);
+    }
+}
+function tokenize(s) {
+    return new Set(s
+        .toLowerCase()
+        .replace(/[^\p{L}\p{N}\s]/gu, " ")
+        .split(/\s+/)
+        .filter((t) => t.length > 2));
+}
+function jaccardSimilarity(a, b) {
+    const ta = tokenize(a);
+    const tb = tokenize(b);
+    if (ta.size === 0 && tb.size === 0)
+        return 1;
+    if (ta.size === 0 || tb.size === 0)
+        return 0;
+    let inter = 0;
+    for (const t of ta)
+        if (tb.has(t))
+            inter += 1;
+    const union = ta.size + tb.size - inter;
+    return union === 0 ? 1 : inter / union;
+}
+function clusterResponses(contestants, threshold) {
+    const ok = contestants.filter((c) => c.status === "ok");
+    if (ok.length === 0)
+        return [];
+    const clusters = [];
+    const cluster_of = new Map();
+    for (const c of ok) {
+        let assigned = -1;
+        for (let i = 0; i < clusters.length; i += 1) {
+            const seedId = clusters[i]?.[0];
+            if (!seedId)
+                continue;
+            const seed = ok.find((x) => x.id === seedId);
+            if (!seed)
+                continue;
+            if (jaccardSimilarity(c.response, seed.response) >= threshold) {
+                assigned = i;
+                break;
+            }
+        }
+        if (assigned === -1) {
+            clusters.push([c.id]);
+            cluster_of.set(c.id, clusters.length - 1);
+        }
+        else {
+            clusters[assigned].push(c.id);
+            cluster_of.set(c.id, assigned);
+        }
+    }
+    return Object.freeze(clusters.map((c) => Object.freeze(c.slice())));
+}
+function clusterWeightsFor(clusters, contestants) {
+    const weights = clusters.map((cluster) => cluster.reduce((sum, id) => {
+        const c = contestants.find((x) => x.id === id);
+        return sum + (c?.weight ?? 1);
+    }, 0));
+    return Object.freeze(weights);
+}
+// ── Voting schemes ───────────────────────────────────────────────────
+function pickLongest(contestants) {
+    const ok = contestants.filter((c) => c.status === "ok");
+    if (ok.length === 0)
+        return null;
+    return ok.reduce((best, c) => (c.response.length > best.response.length ? c : best));
+}
+function pickMostAgreeing(contestants, clusters, weights) {
+    const ok = contestants.filter((c) => c.status === "ok");
+    if (ok.length === 0)
+        return null;
+    if (clusters.length === 0)
+        return ok[0];
+    // Find the cluster with highest weight; pick the contestant with
+    // the highest weight inside that cluster.
+    let bestIdx = 0;
+    let bestW = weights[0] ?? 0;
+    for (let i = 1; i < clusters.length; i += 1) {
+        const w = weights[i] ?? 0;
+        if (w > bestW) {
+            bestW = w;
+            bestIdx = i;
+        }
+    }
+    const cluster = clusters[bestIdx] ?? [];
+    let pick = null;
+    for (const id of cluster) {
+        const c = ok.find((x) => x.id === id);
+        if (!c)
+            continue;
+        if (!pick || c.weight > pick.weight)
+            pick = c;
+    }
+    return pick;
+}
+function pickJudgeScored(contestants) {
+    const ok = contestants.filter((c) => c.status === "ok" && typeof c.judgeScore === "number");
+    if (ok.length === 0)
+        return null;
+    return ok.reduce((best, c) => ((c.judgeScore ?? 0) > (best.judgeScore ?? 0) ? c : best));
+}
+function pickFirstOk(contestants) {
+    return contestants.find((c) => c.status === "ok") ?? null;
+}
+// ── Main API ─────────────────────────────────────────────────────────
+/**
+ * Run an arena. Returns ArenaResult; never throws (every contestant
+ * error is captured into ContestantResult.errorMessage).
+ */
+export async function runArena(req) {
+    const start = Date.now();
+    const timeoutMs = req.timeoutMs ?? 30_000;
+    const concurrency = Math.max(1, req.concurrency ?? req.contestants.length);
+    const voting = req.voting ?? "longest";
+    const clusterThreshold = req.clusterThreshold ?? 0.5;
+    if (voting === "judge-scored" && !req.judge) {
+        throw new Error("voting='judge-scored' requires a judge function");
+    }
+    const results = [];
+    // Bounded-parallel dispatch.
+    for (let i = 0; i < req.contestants.length; i += concurrency) {
+        const batch = req.contestants.slice(i, i + concurrency);
+        const batchResults = await Promise.all(batch.map(async (contestant) => {
+            const t0 = Date.now();
+            const r = await runWithTimeout(() => contestant.fn(req.query), timeoutMs);
+            const latencyMs = Date.now() - t0;
+            return Object.freeze({
+                id: contestant.id,
+                label: contestant.label ?? contestant.id,
+                weight: contestant.weight ?? 1,
+                status: r.status,
+                response: r.response,
+                ...(r.error
+                    ? { errorMessage: r.error }
+                    : r.status === "timeout"
+                        ? { errorMessage: `timeout after ${timeoutMs}ms` }
+                        : {}),
+                latencyMs,
+            });
+        }));
+        results.push(...batchResults);
+    }
+    // Judge-score pass when needed.
+    if (voting === "judge-scored" && req.judge) {
+        const okResults = results.filter((c) => c.status === "ok");
+        const scores = await req.judge({
+            query: req.query,
+            responses: okResults.map((c) => ({ id: c.id, response: c.response })),
+        });
+        const scoreMap = new Map(scores.map((s) => [s.id, s.score]));
+        for (let i = 0; i < results.length; i += 1) {
+            const r = results[i];
+            const score = scoreMap.get(r.id);
+            if (typeof score === "number") {
+                results[i] = Object.freeze({ ...r, judgeScore: Math.max(0, Math.min(1, score)) });
+            }
+        }
+    }
+    const clusters = clusterResponses(results, clusterThreshold);
+    const clusterWeights = clusterWeightsFor(clusters, results);
+    let winner;
+    switch (voting) {
+        case "longest":
+            winner = pickLongest(results);
+            break;
+        case "most-agreeing":
+            winner = pickMostAgreeing(results, clusters, clusterWeights);
+            break;
+        case "judge-scored":
+            winner = pickJudgeScored(results);
+            break;
+        case "first-ok":
+            winner = pickFirstOk(results);
+            break;
+        default:
+            winner = pickLongest(results);
+    }
+    return Object.freeze({
+        query: req.query,
+        contestants: Object.freeze(results),
+        winnerId: winner?.id ?? null,
+        winningResponse: winner?.response ?? null,
+        votingScheme: voting,
+        totalLatencyMs: Date.now() - start,
+        clusters,
+        clusterWeights,
+    });
+}
+// ── Reporting ────────────────────────────────────────────────────────
+/**
+ * Render a Markdown side-by-side report for the arena result. Suitable
+ * for `wotann ow arena run --md`, PR comments, or inline display in
+ * Desktop / TUI surfaces.
+ */
+export function renderArenaMarkdown(result) {
+    const lines = [];
+    lines.push(`# Arena: ${result.query.slice(0, 60)}${result.query.length > 60 ? "..." : ""}`);
+    lines.push("");
+    lines.push(`Winner: **${result.winnerId ?? "(none)"}** · scheme: ${result.votingScheme} · total ${result.totalLatencyMs}ms`);
+    lines.push("");
+    lines.push("| Contestant | Status | Latency | Weight | Score |");
+    lines.push("|---|---|---|---|---|");
+    for (const c of result.contestants) {
+        const score = typeof c.judgeScore === "number" ? c.judgeScore.toFixed(2) : "—";
+        lines.push(`| ${c.label} | ${c.status} | ${c.latencyMs}ms | ${c.weight} | ${score} |`);
+    }
+    lines.push("");
+    if (result.clusters.length > 1) {
+        lines.push(`## Clusters (Jaccard agreement)`);
+        for (let i = 0; i < result.clusters.length; i += 1) {
+            lines.push(`- Cluster ${i + 1} (weight ${result.clusterWeights[i]?.toFixed(2)}): ${(result.clusters[i] ?? []).join(", ")}`);
+        }
+        lines.push("");
+    }
+    for (const c of result.contestants) {
+        if (c.status !== "ok")
+            continue;
+        lines.push(`## ${c.label}`);
+        lines.push("");
+        lines.push("```");
+        lines.push(c.response.slice(0, 4000));
+        lines.push("```");
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+//# sourceMappingURL=multi-model-arena.js.map