terminal-smart-cli 0.94.0 → 0.94.1

package/lib/eval.js

@@ -20,6 +20,7 @@ const fs = require('fs');
 const path = require('path');
 const { api } = require('./api');
 const agent = require('./agent');
+const intelligence = require('./intelligence-core');
 
 // modelo default do JUIZ: barato e suficiente pra avaliar critério objetivo.
 const DEFAULT_JUDGE = 'deepseek-v4-flash';
@@ -163,6 +164,7 @@ async function runSuite(suite, opts = {}) {
     let sandbox = null;
     let evidence = { text: '', error: null, diff: { criados: [], alterados: [], removidos: [] } };
     let runCredits = 0, runSteps = 0, verdict = null, judgeTin = 0, judgeTout = 0;
+    let actualModel = null, substituted = false;
     const t0 = Date.now();
     try {
       // sandbox DENTRO do try: se mkdtempSync falhar (EMFILE/disco cheio/permissão) o caso
@@ -192,17 +194,33 @@ async function runSuite(suite, opts = {}) {
         evidence.error = String((e && e.message) || e).slice(0, 300);
         out = null;
       }
-      if (out) { evidence.text = out.text || ''; runCredits = out.credits || 0; runSteps = out.steps || 0; }
+      if (out) {
+        evidence.text = out.text || '';
+        runCredits = out.credits || 0;
+        runSteps = out.steps || 0;
+        actualModel = out.model || null;
+        substituted = !_sameRequestedModel(model, actualModel);
+        if (substituted) {
+          evidence.error = `model_substituted: requested=${model}; actual=${actualModel || 'unknown'}`;
+          verdict = {
+            pass: false,
+            score: 0,
+            reason: `modelo solicitado "${model}" foi substituído por "${actualModel || 'desconhecido'}"; resultado não é comparável`,
+          };
+        }
+      }
       const after = _snap(sandbox);
       evidence.diff = _diff(before, after);
       // julga (mesmo com erro fatal — o juiz vê o erro e reprova com motivo)
-      onCase({ i, total, id: c.id, phase: 'judge' });
-      try {
-        verdict = await judge({ key: k.key, baseUrl: k.baseUrl, model: judgeModel, caseObj: c, evidence });
-        judgeTin = verdict._tin || 0; judgeTout = verdict._tout || 0;
-      } catch (e) {
-        if (e && (e.code === 'no_credits' || e.status === 402)) throw e;
-        verdict = { pass: false, score: 0, reason: 'juiz falhou: ' + String((e && e.message) || e).slice(0, 120) };
+      if (!substituted) {
+        onCase({ i, total, id: c.id, phase: 'judge' });
+        try {
+          verdict = await judge({ key: k.key, baseUrl: k.baseUrl, model: judgeModel, caseObj: c, evidence });
+          judgeTin = verdict._tin || 0; judgeTout = verdict._tout || 0;
+        } catch (e) {
+          if (e && (e.code === 'no_credits' || e.status === 402)) throw e;
+          verdict = { pass: false, score: 0, reason: 'juiz falhou: ' + String((e && e.message) || e).slice(0, 120) };
+        }
       }
     } catch (e) {
       // teto de IA → aborta a suíte inteira (o finally limpa o sandbox antes de propagar).
@@ -219,6 +237,7 @@ async function runSuite(suite, opts = {}) {
     results.push({
       id: c.id, pass: verdict.pass, score: verdict.score, reason: verdict.reason,
       error: evidence.error, steps: runSteps, credits: runCredits,
+      requestedModel: model, actualModel, substituted,
       changed: evidence.diff.criados.length + evidence.diff.alterados.length,
       judgeTokens: judgeTin + judgeTout, ms: Date.now() - t0, judgeRaw: verdict._raw || null,
     });
@@ -235,6 +254,8 @@ async function runSuite(suite, opts = {}) {
     // exibidos à parte pra o custo do relatório ser honesto).
     judgeTokens: results.reduce((a, r) => a + (r.judgeTokens || 0), 0),
     ms: results.reduce((a, r) => a + (r.ms || 0), 0),
+    actualModels: [...new Set(results.map(r => r.actualModel).filter(Boolean))],
+    substitutions: results.filter(r => r.substituted).length,
   };
   return { summary, results };
 }
@@ -242,6 +263,12 @@ async function runSuite(suite, opts = {}) {
 // ── MODEL LAB: roda a MESMA suíte em VÁRIOS executores e compara (bake-off automático) ──
 // Automatiza o "veredito modelo X" que antes era manual. O JUIZ é fixo (mesmo pra todos =
 // comparação justa); só o EXECUTOR muda. Ranqueia por qualidade VERIFICADA e depois por custo.
+function _sameRequestedModel(requested, actual) {
+  if (!requested) return true;
+  if (!actual) return false;
+  return intelligence.normalizeModelId(requested) === intelligence.normalizeModelId(actual);
+}
+
 function _rankModels(rows) {
   // ordena: mais casos aprovados (passRate) → maior score médio → MENOR custo → mais rápido.
   const sorted = [...rows].sort((a, b) =>
@@ -264,7 +291,12 @@ async function runMatrix(suite, opts = {}) {
       if (e && (e.code === 'no_credits' || e.status === 402)) throw e; // teto → aborta o bake-off
       sum = { name: suite.name, total: suite.cases.length, passed: 0, failed: suite.cases.length, passRate: 0, avgScore: 0, credits: 0, judgeTokens: 0, ms: 0, error: String((e && e.message) || e).slice(0, 160) };
     }
-    const row = { model, passRate: sum.passRate, avgScore: sum.avgScore, passed: sum.passed, total: sum.total, credits: sum.credits, judgeTokens: sum.judgeTokens, ms: sum.ms, error: sum.error || null };
+    const row = {
+      model, passRate: sum.passRate, avgScore: sum.avgScore, passed: sum.passed, total: sum.total,
+      credits: sum.credits, judgeTokens: sum.judgeTokens, ms: sum.ms,
+      actualModels: sum.actualModels || [], substitutions: sum.substitutions || 0,
+      error: sum.error || null,
+    };
     rows.push(row);
     onModel({ model, phase: 'done', row });
   }
@@ -273,4 +305,4 @@ async function runMatrix(suite, opts = {}) {
 }
 
 module.exports = { loadSuite, normCases, runSuite, runMatrix, judge, smokeSuite, EXAMPLE,
-  _test: { _diff, _snap, _extractJson, normCases, _rankModels } };
+  _test: { _diff, _snap, _extractJson, normCases, _rankModels, _sameRequestedModel } };

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "terminal-smart-cli",
-  "version": "0.94.0",
+  "version": "0.94.1",
   "description": "Terminal Smart no seu terminal — pergunte, analise logs por pipe e orquestre agentes de IA. Comando: ts",
   "bin": {
     "ts": "bin/ts.js"
   },
   "scripts": {
-    "test": "node test/core.test.js && node test/intelligence-core.test.js && node test/project-cache.test.js && node test/memory-bus.test.js && node test/capabilities.test.js && node test/mcp-e2e.test.js && node test/erros.test.js"
+    "test": "node test/core.test.js && node test/intelligence-core.test.js && node test/eval-model.test.js && node test/project-cache.test.js && node test/memory-bus.test.js && node test/capabilities.test.js && node test/mcp-e2e.test.js && node test/erros.test.js"
   },
   "files": [
     "bin",