winter-super-cli 2026.5.27 → 2026.5.29

package/bin/winter.js

@@ -17,7 +17,7 @@ const pkg = JSON.parse(readFileSync(new URL('../package.json', import.meta.url),
 const version = pkg.version;
 
 const COMMANDS = new Set([
-  'chat', 'call', 'session', 'skill', 'plugin', 'design', 'config', 'init',
+  'chat', 'call', 'benchmark', 'session', 'skill', 'plugin', 'design', 'config', 'init',
   'help', 'project', 'code', 'review', 'mcp', 'permissions',
   'provider', 'providers', 'model', 'models',
 ]);
@@ -53,6 +53,7 @@ Usage:
 Commands:
 
   winter call <prompt>        Call all configured providers
+  winter benchmark [providers] Benchmark model intelligence
   winter session <action>     Session management
   winter skill <action>       Skill management
   winter plugin <action>      Plugin management

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "winter-super-cli",
-  "version": "2026.5.27",
+  "version": "2026.5.29",
   "description": "❄️ AI-Powered Development CLI with Interactive REPL",
   "type": "module",
   "main": "bin/winter.js",

package/src/ai/benchmark.js

@@ -0,0 +1,352 @@
+/**
+ * Benchmark Engine — Đo độ thông minh của models trong Winter CLI
+ *
+ * Cố định câu hỏi test (logic, coding, math, reasoning, language)
+ * + Coding task thật → chạy qua providers → chấm điểm → so sánh
+ */
+
+import { colors } from '../cli/snowflake-logo.js';
+
+// ── Question Bank ────────────────────────────────────────────────────────────
+
+const BENCHMARK_QUESTIONS = [
+  {
+    id: 'q01',
+    category: 'logic',
+    question: `If all cats are mammals and some mammals are dogs, are all cats dogs? Explain your reasoning step by step.`,
+    keywords: ['not', 'no', 'incorrect', 'cannot conclude', 'not necessarily', 'invalid'],
+    weight: 1,
+  },
+  {
+    id: 'q02',
+    category: 'coding',
+    question: `Write a JavaScript function called isPalindrome that checks if a string is a palindrome (reads the same forwards and backwards). Include example usage.`,
+    keywords: ['function', 'palindrome', 'reverse', 'split', 'return'],
+    weight: 1.5,
+  },
+  {
+    id: 'q03',
+    category: 'math',
+    question: `What is 15% of 200? Show your calculation.`,
+    keywords: ['30', '15', '200', '0.15'],
+    weight: 0.5,
+  },
+  {
+    id: 'q04',
+    category: 'reasoning',
+    question: `A bat and a ball cost $1.10 in total. The bat costs $1.00 more than the ball. How much does the ball cost? Think carefully.`,
+    keywords: ['0.05', '5 cents', '5 cent', '0.05$', '5¢', 'five cents'],
+    weight: 1.5,
+  },
+  {
+    id: 'q05',
+    category: 'coding',
+    question: `What's wrong with this code and how would you fix it?\n\nfunction add(a, b) {\n  return a + b;\n}\nconsole.log(add(5, '3'));`,
+    keywords: ['string', 'type', 'concatenation', 'number', 'parse', 'typeof', 'coercion'],
+    weight: 1,
+  },
+  {
+    id: 'q06',
+    category: 'language',
+    question: `Translate this sentence to Vietnamese: "Good morning, how are you today?"`,
+    keywords: ['chào', 'sáng', 'khỏe', 'hôm nay', 'bạn'],
+    weight: 0.5,
+  },
+  {
+    id: 'q07',
+    category: 'logic',
+    question: `You have a 3-gallon jug and a 5-gallon jug. How can you measure exactly 4 gallons of water? Explain step by step.`,
+    keywords: ['fill', 'pour', '3', '5', '4', 'empty'],
+    weight: 1.5,
+  },
+  {
+    id: 'q08',
+    category: 'coding',
+    question: `Write a recursive function to calculate the nth Fibonacci number. Explain how memoization can optimize it.`,
+    keywords: ['function', 'fibonacci', 'recursive', 'memoization', 'cache'],
+    weight: 1.5,
+  },
+];
+
+const BENCHMARK_TASKS = [
+  {
+    id: 't01',
+    category: 'coding-task',
+    title: 'API Fetch with Error Handling',
+    description: 'Write a JavaScript function that fetches JSON data from a URL, handles network errors, HTTP errors, and invalid JSON responses gracefully.',
+    evaluationCriteria: ['error handling', 'try/catch', 'async/await', 'fetch', 'response.ok'],
+    weight: 2,
+  },
+  {
+    id: 't02',
+    category: 'coding-task',
+    title: 'Event Emitter Class',
+    description: 'Create a simple EventEmitter class in JavaScript with on(), off(), and emit() methods. It should support multiple listeners for the same event and removing listeners.',
+    evaluationCriteria: ['class', 'on', 'off', 'emit', 'listeners', 'events'],
+    weight: 2,
+  },
+  {
+    id: 't03',
+    category: 'coding-task',
+    title: 'Fix This Bug',
+    description: `What's wrong with this code? Identify ALL bugs and provide a fixed version:\n\nconst users = [\n  { name: 'Alice', age: 30 },\n  { name: 'Bob', age: 25 },\n  { name: 'Charlie', age: 35 },\n];\n\nconst adultUsers = users.filter(u => u.age >= 18);\nadultUsers.forEach(u => {\n  console.log(u.Name);\n});\n\nadultUsers.sort((a, b) => a.age - b.age);\nconst totalAge = adultUsers.reduce((acc, u) => acc + u.age);\nconsole.log('Average age:', totalAge / adultUsers.length);`,
+    evaluationCriteria: ['Name', 'name', 'undefined', 'reduce', 'initial', 'initialize', 'capital N'],
+    weight: 2.5,
+  },
+];
+
+// ── Scoring ──────────────────────────────────────────────────────────────────
+
+function scoreAnswer(question, answer) {
+  if (!answer || typeof answer !== 'string') return 0;
+
+  const lower = answer.toLowerCase();
+  let matches = 0;
+
+  for (const kw of question.keywords) {
+    if (lower.includes(kw.toLowerCase())) {
+      matches++;
+    }
+  }
+
+  const ratio = question.keywords.length > 0 ? matches / question.keywords.length : 0;
+
+  // Bonus: longer, well-structured answers tend to be better
+  const words = answer.split(/\s+/).length;
+  const lengthBonus = words > 50 ? 0.1 : words > 20 ? 0.05 : 0;
+
+  return Math.min(1, ratio + lengthBonus);
+}
+
+function scoreTask(task, answer) {
+  if (!answer || typeof answer !== 'string') return 0;
+
+  const lower = answer.toLowerCase();
+  let matches = 0;
+
+  for (const criterion of task.evaluationCriteria) {
+    if (lower.includes(criterion.toLowerCase())) {
+      matches++;
+    }
+  }
+
+  const ratio = task.evaluationCriteria.length > 0 ? matches / task.evaluationCriteria.length : 0;
+  const words = answer.split(/\s+/).length;
+  const lengthBonus = words > 100 ? 0.1 : words > 50 ? 0.05 : 0;
+
+  return Math.min(1, ratio + lengthBonus);
+}
+
+// ── Benchmark Runner ─────────────────────────────────────────────────────────
+
+export class BenchmarkRunner {
+  constructor(aiManager) {
+    this.ai = aiManager;
+  }
+
+  /**
+   * Run all benchmark questions across specified providers.
+   * @param {string[]} providerNames - List of provider names (e.g., ['claude', 'openai', 'ollama'])
+   * @param {object} options
+   * @param {boolean} options.tasks - Whether to include coding tasks (default: true)
+   * @param {boolean} options.questions - Whether to include fixed questions (default: true)
+   */
+  async run(providerNames, options = {}) {
+    const { questions = true, tasks = true } = options;
+
+    await this.ai.init();
+
+    // Filter to only ready providers
+    const providers = providerNames
+      .map(name => ({ name, provider: this.ai.providers[name] }))
+      .filter(({ provider }) => provider && provider.ready);
+
+    if (providers.length === 0) {
+      return { error: 'No ready providers found. Configure providers in winter.json first.' };
+    }
+
+    const results = {};
+    const startTime = Date.now();
+
+    for (const { name, provider } of providers) {
+      console.log(`${colors.dim}Benchmarking ${colors.bright}${name}${colors.reset}${colors.dim}...${colors.reset}`);
+
+      const providerResults = [];
+      let totalScore = 0;
+      let maxScore = 0;
+
+      // Fixed questions
+      if (questions) {
+        for (const q of BENCHMARK_QUESTIONS) {
+          const qStart = Date.now();
+          const answer = await this.askProvider(provider, q.question);
+          const elapsed = Date.now() - qStart;
+          const score = scoreAnswer(q, answer);
+
+          providerResults.push({
+            type: 'question',
+            id: q.id,
+            category: q.category,
+            question: q.question,
+            answer: answer.slice(0, 500), // truncate for display
+            score,
+            weightedScore: score * q.weight,
+            maxWeightedScore: q.weight,
+            elapsed,
+          });
+
+          totalScore += score * q.weight;
+          maxScore += q.weight;
+        }
+      }
+
+      // Coding tasks
+      if (tasks) {
+        for (const t of BENCHMARK_TASKS) {
+          const tStart = Date.now();
+          const answer = await this.askProvider(provider, t.description);
+          const elapsed = Date.now() - tStart;
+          const score = scoreTask(t, answer);
+
+          providerResults.push({
+            type: 'task',
+            id: t.id,
+            category: t.category,
+            title: t.title,
+            question: t.description,
+            answer: answer.slice(0, 500),
+            score,
+            weightedScore: score * t.weight,
+            maxWeightedScore: t.weight,
+            elapsed,
+          });
+
+          totalScore += score * t.weight;
+          maxScore += t.weight;
+        }
+      }
+
+      const overall = maxScore > 0 ? Math.round((totalScore / maxScore) * 100) : 0;
+
+      results[name] = {
+        provider: name,
+        model: provider.model,
+        results: providerResults,
+        totalScore,
+        maxScore,
+        overall,
+        elapsed: Date.now() - startTime,
+      };
+    }
+
+    return {
+      timestamp: new Date().toISOString(),
+      totalElapsed: Date.now() - startTime,
+      providers: results,
+      // Sort providers by overall score descending
+      ranking: Object.values(results)
+        .sort((a, b) => b.overall - a.overall)
+        .map(r => ({ name: r.provider, model: r.model, score: r.overall, elapsed: r.elapsed })),
+    };
+  }
+
+  async askProvider(provider, prompt) {
+    try {
+      const messages = [
+        { role: 'system', content: 'You are a helpful AI assistant. Answer concisely and accurately.' },
+        { role: 'user', content: prompt },
+      ];
+      const data = await this.ai.sendRequestToProvider(provider, messages, {
+        enableTools: false,
+        model: provider.model,
+      });
+      return data.choices?.[0]?.message?.content || '';
+    } catch (err) {
+      return `[ERROR: ${err.message}]`;
+    }
+  }
+
+  // ── Format Results ────────────────────────────────────────────────────────
+
+  formatResults(benchmarkResult) {
+    if (benchmarkResult.error) {
+      return `\n${colors.red}${benchmarkResult.error}${colors.reset}\n`;
+    }
+
+    const lines = [];
+    lines.push(`\n${colors.cyan}${'═'.repeat(60)}${colors.reset}`);
+    lines.push(`${colors.bright}${colors.cyan}   🧠 WINTER MODEL BENCHMARK${colors.reset}`);
+    lines.push(`${colors.cyan}${'═'.repeat(60)}${colors.reset}`);
+    lines.push(`  ${colors.dim}${benchmarkResult.timestamp}${colors.reset}`);
+    lines.push(`  ${colors.dim}Total time: ${(benchmarkResult.totalElapsed / 1000).toFixed(1)}s${colors.reset}`);
+    lines.push('');
+
+    // Ranking
+    lines.push(`${colors.bright}🏆 RANKING${colors.reset}`);
+    lines.push(`${'─'.repeat(40)}`);
+    benchmarkResult.ranking.forEach((r, i) => {
+      const medal = i === 0 ? '🥇' : i === 1 ? '🥈' : i === 2 ? '🥉' : ` ${i + 1}.`;
+      const bar = this._scoreBar(r.score, 20);
+      lines.push(`  ${medal} ${colors.bright}${r.name}${colors.reset} ${bar} ${r.score}%`);
+      lines.push(`     ${colors.dim}Model: ${r.model} | Time: ${(r.elapsed / 1000).toFixed(1)}s${colors.reset}`);
+    });
+    lines.push('');
+
+    // Detail per provider
+    for (const [name, data] of Object.entries(benchmarkResult.providers)) {
+      lines.push(`${colors.bright}${'─'.repeat(50)}${colors.reset}`);
+      lines.push(`${colors.bright}📊 ${name}${colors.reset} ${colors.dim}(${data.model})${colors.reset}`);
+      lines.push(`${'─'.repeat(50)}`);
+
+      const categories = {};
+      for (const r of data.results) {
+        const cat = r.category || 'other';
+        if (!categories[cat]) categories[cat] = { count: 0, totalScore: 0, maxScore: 0 };
+        categories[cat].count++;
+        categories[cat].totalScore += r.score;
+        categories[cat].maxScore += 1;
+      }
+
+      for (const [cat, stats] of Object.entries(categories)) {
+        const catPct = Math.round((stats.totalScore / stats.maxScore) * 100);
+        const bar = this._scoreBar(catPct, 10);
+        lines.push(`  ${bar} ${colors.dim}${cat}:${colors.reset} ${catPct}% (${stats.count} items)`);
+      }
+      lines.push('');
+
+      // Per-item breakdown
+      for (const r of data.results) {
+        const icon = r.score >= 0.8 ? '✅' : r.score >= 0.5 ? '🟡' : r.score >= 0.2 ? '🟠' : '❌';
+        const label = r.type === 'question' ? r.id : r.title;
+        lines.push(`  ${icon} ${colors.dim}${label}:${colors.reset} ${Math.round(r.score * 100)}% (${(r.elapsed / 1000).toFixed(1)}s)`);
+        // Show preview of answer
+        const preview = r.answer.replace(/\n/g, ' ').slice(0, 120);
+        lines.push(`    ${colors.dim}${preview}${r.answer.length > 120 ? '...' : ''}${colors.reset}`);
+      }
+      lines.push('');
+    }
+
+    lines.push(`${colors.cyan}${'═'.repeat(60)}${colors.reset}\n`);
+
+    return lines.join('\n');
+  }
+
+  _scoreBar(score, width = 20) {
+    const filled = Math.round((score / 100) * width);
+    const empty = width - filled;
+    const filledChar = '█';
+    const emptyChar = '░';
+    return colors.green + filledChar.repeat(filled) + colors.dim + emptyChar.repeat(empty) + colors.reset;
+  }
+
+  // ── History ───────────────────────────────────────────────────────────────
+
+  formatHistorySummary(benchmarkResult) {
+    return benchmarkResult.ranking
+      .map(r => `[${r.name}] Score: ${r.score}% | Model: ${r.model} | Time: ${(r.elapsed / 1000).toFixed(1)}s`)
+      .join('\n');
+  }
+}
+
+// Export question/task banks for testing
+export { BENCHMARK_QUESTIONS, BENCHMARK_TASKS, scoreAnswer, scoreTask };

package/src/ai/model-capabilities.js

@@ -0,0 +1,185 @@
+/**
+ * ❄️ MODEL CAPABILITIES ❄️
+ * Detect AI model capability tier from model name.
+ * Small models need aggressive prompting to compete with large ones.
+ */
+
+export const MODEL_TIERS = {
+  TINY: 'tiny',      // <3B params — barely functional for code
+  SMALL: 'small',    // 3B-15B params — basic code ability
+  MEDIUM: 'medium',  // 15B-40B params — decent code ability
+  LARGE: 'large',    // 40B-120B params — strong code ability, could be flagship
+  FLAGSHIP: 'flagship', // 120B+ or proprietary frontier models
+};
+
+/**
+ * Ordered tiers from weakest to strongest (for comparison).
+ */
+const TIER_ORDER = [MODEL_TIERS.TINY, MODEL_TIERS.SMALL, MODEL_TIERS.MEDIUM, MODEL_TIERS.LARGE, MODEL_TIERS.FLAGSHIP];
+
+/**
+ * Classify a model name into a capability tier.
+ * @param {string} modelName - e.g. "llama3", "gpt-4", "qwen2.5:7b"
+ * @param {string} [provider] - e.g. "ollama", "openai" (optional, helps disambiguate)
+ * @returns {string} One of MODEL_TIERS
+ */
+export function classifyModelTier(modelName, provider = '') {
+  const name = (modelName || '').toLowerCase().trim();
+  const prov = (provider || '').toLowerCase().trim();
+
+  // ===== FLAGSHIP (frontier models) =====
+  const flagshipPatterns = [
+    /claude-3-5-sonnet/i, /claude-opus/i, /claude-4/i, /claude-sonnet-4/i,
+    /gpt-4o/i, /gpt-4-turbo/i, /o1/i, /o3/i,
+    /gemini-2\.5-pro/i, /gemini-2\.0-ultra/i,
+    /deepseek-v3/i, /deepseek-r1/i,
+    /llama-4/i, /llama-3-70b/i, /llama3-70b/i, /llama3\.1-70b/i, /llama3\.2-90b/i, /llama3\.3/i,
+    /qwen2\.5-?72b/i, /qwen2\.5-?70b/i, /qwen-?2\.5-?72b/i,
+    /mistral-large/i, /mixtral-8x22b/i,
+    /command-r-plus/i, /command-a/i,
+    /yi-?34b/i,
+    /dbrx-instruct/i,
+  ];
+
+  // If using a cloud provider like OpenAI/Anthropic/Groq, their default models are typically large+
+  if (prov === 'openai' || prov === 'anthropic' || prov === 'claude') {
+    if (name.includes('gpt-3.5') || name.includes('gpt-3')) return MODEL_TIERS.MEDIUM;
+    if (name.includes('claude-3-haiku') || name.includes('claude-3-5-haiku')) return MODEL_TIERS.MEDIUM;
+    return MODEL_TIERS.LARGE; // Default for OpenAI/Anthropic is >= gpt-4 level
+  }
+
+  if (prov === 'groq') {
+    // Groq runs open models, most are large but some are not
+    if (/llama.*8b|llama3.*8b|llama3\.2.*3b/i.test(name)) return MODEL_TIERS.SMALL;
+    if (/gemma2.*9b/i.test(name)) return MODEL_TIERS.SMALL;
+    if (/mixtral-8x7|llama.*70b|llama3.*70b|llama3\.1.*70b|qwen/i.test(name)) return MODEL_TIERS.LARGE;
+    return MODEL_TIERS.MEDIUM; // Default for Groq
+  }
+
+  // Check patterns for any provider
+  for (const pattern of flagshipPatterns) {
+    if (pattern.test(name)) return MODEL_TIERS.FLAGSHIP;
+  }
+
+  // ===== LARGE MODELS =====
+  const largePatterns = [
+    /claude-sonnet/i, /claude-3/i, /claude-2/i,
+    /gpt-4/i, /gpt-4-32k/i,
+    /llama-3\.1-?70b/i, /llama-3\.2-?70b/i, /llama3-?70b/i,
+    /llama-2-?70b/i,
+    /qwen-?2\.5-?32b/i, /qwen-?2-?72b/i,
+    /codellama-?70b/i,
+    /mixtral/i,
+    /deepseek-?v2/i,
+    /gemini-1\.5-pro/i, /gemini-2\.0-flash/i,
+    /command-r/i,
+    /yi-?34b/i,
+    /mistral-medium/i,
+  ];
+
+  for (const pattern of largePatterns) {
+    if (pattern.test(name)) return MODEL_TIERS.LARGE;
+  }
+
+  // ===== MEDIUM MODELS =====
+  const mediumPatterns = [
+    /qwen-?2\.5-?14b/i, /qwen-?2\.5-?7b/i, /qwen-?2/i,
+    /llama-3-?8b/i, /llama-3\.1-?8b/i, /llama-3\.2-?11b/i,
+    /llama-2-?13b/i, /llama-2-?7b/i,
+    /deepseek-coder-?6\.7b/i, /deepseek-coder-?33b/i,
+    /codellama-?34b/i, /codellama-?13b/i, /codellama-?7b/i,
+    /mistral/i, /mistral-7b/i,
+    /gemma-2-?9b/i, /gemma-?7b/i,
+    /phi-3/i, /phi-3-medium/i,
+    /nemotron/i,
+    /solar/i,
+    /dbrx/i,
+    /starcoder2/i,
+    /deepseek-llm/i,
+    /yi-?6b/i, /yi-?9b/i,
+  ];
+
+  for (const pattern of mediumPatterns) {
+    if (pattern.test(name)) return MODEL_TIERS.MEDIUM;
+  }
+
+  // ===== SMALL MODELS =====
+  const smallPatterns = [
+    /llama-3\.2-?3b/i, /llama-3\.2-?1b/i, /tinyllama/i,
+    /qwen-?2\.5-?3b/i, /qwen-?2\.5-?1\.5b/i, /qwen-?2\.5-?0\.5b/i,
+    /phi-?3-?mini/i, /phi-?2/i, /phi-?1/i,
+    /gemma-?2-?2b/i,
+    /stablelm/i,
+    /orca/i,
+    /falcon/i,
+    /red-pajama/i,
+    /pythia/i,
+    /opt/i,
+    /bloom/i,
+    /mpnet/i,
+  ];
+
+  for (const pattern of smallPatterns) {
+    if (pattern.test(name)) return MODEL_TIERS.SMALL;
+  }
+
+  if (/tiny/i.test(name) || /mini/i.test(name) || /small/i.test(name) || /nano/i.test(name)) {
+    return MODEL_TIERS.TINY;
+  }
+
+  // Fallback: if Ollama, likely small
+  if (prov === 'ollama' || prov === 'local') return MODEL_TIERS.SMALL;
+
+  // Default: assume medium
+  return MODEL_TIERS.MEDIUM;
+}
+
+/**
+ * Check if a model tier is considered "small" (needs aggressive prompting).
+ */
+export function isSmallModel(tier) {
+  return tier === MODEL_TIERS.TINY || tier === MODEL_TIERS.SMALL;
+}
+
+/**
+ * Get the index of a tier in the order array (0=weakest).
+ * @private
+ */
+function tierIndex(tier) {
+  const idx = TIER_ORDER.indexOf(tier);
+  return idx >= 0 ? idx : 2; // Default to medium index
+}
+
+/**
+ * Compare two tiers. Returns negative if a < b, positive if a > b, 0 if equal.
+ * @private
+ */
+function compareTiers(a, b) {
+  return tierIndex(a) - tierIndex(b);
+}
+
+/**
+ * Get recommended reasoning level bump for small models.
+ * Small models need more aggressive reasoning prompting to compensate.
+ */
+export function getReasoningBump(tier) {
+  switch (tier) {
+    case MODEL_TIERS.TINY: return 2;     // bump 2 levels
+    case MODEL_TIERS.SMALL: return 1;    // bump 1 level
+    default: return 0;
+  }
+}
+
+/**
+ * Build a short string describing model capability for system prompt injection.
+ */
+export function getModelCapabilityLabel(tier) {
+  switch (tier) {
+    case MODEL_TIERS.TINY: return 'tiny local model — needs maximum guidance';
+    case MODEL_TIERS.SMALL: return 'small local model — needs extra structure';
+    case MODEL_TIERS.MEDIUM: return 'medium-capability model';
+    case MODEL_TIERS.LARGE: return 'high-capability model';
+    case MODEL_TIERS.FLAGSHIP: return 'frontier model — full capability expected';
+    default: return '';
+  }
+}