savant-code 0.0.19 → 0.0.20

package/evals/benchmark/lessons-extractor.ts

@@ -1,247 +0,0 @@
-import fs from 'fs'
-import path from 'path'
-
-import { getErrorObject } from '@savant-code/common/util/error'
-
-import { truncateTrace } from './trace-utils'
-
-import type { AgentStep } from './agent-runner'
-import type { JudgingResult } from './judge'
-import type { FileDiff } from './types'
-import type { AgentDefinition, SavantCodeClient } from '@savant-code/sdk'
-
-export interface Lesson {
-  whatWentWrong: string
-  whatShouldHaveBeenDone: string
-}
-
-type ExtractAgentLessonsInput = {
-  client: SavantCodeClient
-  localAgentDefinitions: AgentDefinition[]
-  prompt: string
-  groundTruthFileDiffs: FileDiff[]
-  contextFiles: Record<string, string>
-  agentDiff: string
-  agentTrace: AgentStep[]
-  judgeResult?: JudgingResult
-  error?: string
-}
-
-const lessonsExtractorAgent: AgentDefinition = {
-  id: 'benchmark-lessons-extractor',
-  displayName: 'Benchmark Lessons Extractor',
-  model: 'openai/gpt-5',
-  toolNames: ['spawn_agents', 'read_files', 'set_output'],
-  spawnableAgents: ['scout', 'find-all-referencer'],
-  inputSchema: {
-    prompt: { type: 'string', description: 'Lessons extraction prompt' },
-  },
-  outputMode: 'structured_output',
-  outputSchema: {
-    type: 'object',
-    properties: {
-      lessons: {
-        type: 'array',
-        items: {
-          type: 'object',
-          properties: {
-            whatWentWrong: { type: 'string' },
-            whatShouldHaveBeenDone: { type: 'string' },
-          },
-          required: ['whatWentWrong', 'whatShouldHaveBeenDone'],
-        },
-        description:
-          'Lessons learned from this task. Each lesson should identify what went wrong and what should have been done instead.',
-      },
-    },
-    required: ['lessons'],
-  },
-  systemPrompt: `You are a Lesson Extractor. Your job: analyze agent performance and extract actionable lessons.
-
-Context you receive:
-- User prompt (what the coding agent was asked)
-- Context files from parent commit (relevant code/config from the repo)
-- Ground truth diffs (one valid solution path)
-- The agent's diffs (what they actually changed)
-- A truncated agent trace showing HOW they worked (tools, order, decisions)
-- Optional judge summary (scores, weaknesses)
-
-Use tools to gather more context if needed.
-
-You must output an array of lessons. Each lesson has two parts:
-
-1. **whatWentWrong**: What the agent did incorrectly, misunderstood, or failed to do on this task
-   - Be specific about the mistake or gap in understanding
-   - Reference specific files, functions, or patterns when relevant
-   - Focus on concrete, observable issues
-
-2. **whatShouldHaveBeenDone**: The correct approach the agent should have taken
-   - Provide actionable guidance that would have led to a better result
-   - Be specific and prescriptive
-   - Make it something the agent could apply in the future
-
-Rules:
-- Each lesson should be a complete learning unit (problem + solution)
-- Keep lessons terse but precise (aim for ~140 chars per field if possible)
-- Do not include things the agent already did correctly
-- Focus on gaps that, if filled, would have improved the outcome
-- Don't include repeat lessons or ones that are similar to each other`,
-}
-
-export async function extractAgentLessons(
-  input: ExtractAgentLessonsInput,
-): Promise<{
-  lessons: Lesson[]
-}> {
-  try {
-    const {
-      client,
-      localAgentDefinitions,
-      prompt,
-      groundTruthFileDiffs,
-      contextFiles,
-      agentDiff,
-      agentTrace,
-      judgeResult,
-      error,
-    } = input
-
-    const groundTruthDiffs = groundTruthFileDiffs
-      .map(({ path, diff }) => `### ${path}\n\`\`\`diff\n${diff}\n\`\`\``)
-      .join('\n\n')
-
-    const contextFilesContent = Object.entries(contextFiles)
-      .map(
-        ([filePath, content]) => `### ${filePath}\n\`\`\`\n${content}\n\`\`\``,
-      )
-      .join('\n\n')
-
-    const truncated = truncateTrace(agentTrace)
-    const truncatedTraceJson = JSON.stringify(truncated, null, 2)
-
-    const judgeSummary = judgeResult
-      ? `Overall: ${judgeResult.overallScore}/10, Completion: ${judgeResult.completionScore}/10, Code Quality: ${judgeResult.codeQualityScore}/10
-Weaknesses: ${judgeResult.weaknesses?.join(' | ') || '(none)'}`
-      : '(no judge summary)'
-
-    const lessonsPrompt = `## User Prompt
-${prompt}
-
-## Context Files (from parent commit)
-${contextFilesContent || '(No context files)'}
-
-## Ground Truth Changes (One valid implementation)
-${groundTruthDiffs || '(No ground truth diffs)'}
-
-## Agent's Changes
-\`\`\`diff
-${agentDiff || '(No changes made)'}
-\`\`\`
-
-## Agent Trace (truncated)
-\`\`\`json
-${truncatedTraceJson}
-\`\`\`
-
-## Judge Summary
-${judgeSummary}
-${error ? `\n## Agent Error\n${error}\n` : ''}
-
-Task: Analyze what went wrong and what should have been done. For each mistake or gap, provide a lesson with both parts: whatWentWrong and whatShouldHaveBeenDone. Output the lessons array as specified.`
-
-    const agentOutput: string[] = []
-    const result = await client.run({
-      agent: 'benchmark-lessons-extractor',
-      prompt: lessonsPrompt,
-      agentDefinitions: [lessonsExtractorAgent, ...localAgentDefinitions],
-      // FID-2026-0803-007 EV-3 (implementation note): same withTimeout pattern
-      // found at this site during implementation — abort instead of racing.
-      signal: AbortSignal.timeout(20 * 60 * 1000),
-      handleEvent: (event) => {
-        if (event.type === 'text') agentOutput.push(event.text)
-        else if (event.type === 'tool_call')
-          agentOutput.push(JSON.stringify(event))
-      },
-    })
-
-    if (result.output.type !== 'structuredOutput' || !result.output.value) {
-      console.warn(
-        'Lessons extractor did not return structured output:\n',
-        JSON.stringify(result.output, null, 2),
-      )
-      return { lessons: [] }
-    }
-
-    console.log('Agent output:', agentOutput.join('\n'))
-
-    const output = result.output.value as Record<string, unknown>
-    const rawLessons = output.lessons
-    const lessons: Lesson[] = Array.isArray(rawLessons)
-      ? rawLessons
-          .filter(
-            (item): item is Record<string, unknown> =>
-              typeof item === 'object' && item !== null && !Array.isArray(item),
-          )
-          .map((item) => ({
-            whatWentWrong:
-              typeof item.whatWentWrong === 'string' ? item.whatWentWrong : '',
-            whatShouldHaveBeenDone:
-              typeof item.whatShouldHaveBeenDone === 'string'
-                ? item.whatShouldHaveBeenDone
-                : '',
-          }))
-      : []
-    return { lessons }
-  } catch (error) {
-    console.error('Failed to extract agent lessons:', getErrorObject(error))
-    return { lessons: [] }
-  }
-}
-
-export function saveAgentLessons(params: {
-  agentId: string
-  commitId: string
-  commitSha: string
-  prompt: string
-  lessons: Lesson[]
-  lessonsDir: string
-}): void {
-  const { agentId, commitId, commitSha, prompt, lessons, lessonsDir } = params
-
-  try {
-    const safeAgentId = agentId.replace(/[^a-zA-Z0-9-]/g, '_')
-    if (!fs.existsSync(lessonsDir)) {
-      fs.mkdirSync(lessonsDir, { recursive: true })
-    }
-    const lessonsFile = path.join(lessonsDir, `${safeAgentId}.md`)
-
-    if (!fs.existsSync(lessonsFile)) {
-      fs.writeFileSync(
-        lessonsFile,
-        `# Agent Lessons: ${agentId}\n\nLessons accumulated across benchmark runs. Each lesson identifies what went wrong and what should have been done instead.\n\n`,
-        'utf-8',
-      )
-    }
-
-    if (lessons.length > 0) {
-      const header = `## ${new Date().toISOString()} — ${commitId} (${commitSha.slice(0, 7)})\n`
-      let content = `\n### Original Agent Prompt\n${prompt}\n`
-
-      content += '\n### Lessons\n'
-      content += lessons
-        .map((lesson) => {
-          return `- **What went wrong:** ${lesson.whatWentWrong}\n  **What should have been done:** ${lesson.whatShouldHaveBeenDone}`
-        })
-        .join('\n\n')
-      content += '\n'
-
-      fs.appendFileSync(lessonsFile, `${header}${content}\n`, 'utf-8')
-    }
-  } catch (e) {
-    const error = e instanceof Error ? e : new Error(String(e))
-    console.warn(
-      `Failed to append agent lessons for ${agentId}:`,
-      error.message,
-    )
-  }
-}

package/evals/benchmark/main-hard-tasks.ts

@@ -1,49 +0,0 @@
-import fs from 'fs'
-import path from 'path'
-
-import { runBenchmark } from './run-benchmark'
-
-import type { EvalDataV2 } from './types'
-
-// Load task IDs from an eval file
-function loadTaskIds(evalPath: string): string[] {
-  const content = fs.readFileSync(evalPath, 'utf-8')
-  const evalData: EvalDataV2 = JSON.parse(content)
-  return evalData.evalCommits.map((commit) => commit.id)
-}
-
-async function main() {
-  const saveTraces = process.argv.includes('--save-traces')
-
-  const evalPaths = [
-    path.join(__dirname, 'eval-codebuff2.json'),
-    path.join(__dirname, 'eval-manifold2.json'),
-    path.join(__dirname, 'eval-plane2.json'),
-    path.join(__dirname, 'eval-saleor2.json'),
-  ]
-
-  // Load all task IDs from the eval files
-  const allTaskIds = evalPaths.flatMap(loadTaskIds)
-
-  console.log(
-    `Running ${allTaskIds.length} hard tasks across ${evalPaths.length} eval sets`,
-  )
-
-  // Run all hard tasks across all 4 eval sets
-  await runBenchmark({
-    evalDataPaths: evalPaths,
-    agents: ['savant', 'external:claude'],
-    taskIds: allTaskIds,
-    taskConcurrency: 4,
-    saveTraces,
-  })
-
-  process.exit(0)
-}
-
-if (import.meta.main) {
-  main().catch((error) => {
-    console.error('Error running hard tasks evaluation:', error)
-    process.exit(1)
-  })
-}

package/evals/benchmark/main-single-eval.ts

@@ -1,23 +0,0 @@
-import path from 'path'
-
-import { runBenchmark } from './run-benchmark'
-
-async function main() {
-  const saveTraces = process.argv.includes('--save-traces')
-
-  await runBenchmark({
-    evalDataPaths: [path.join(__dirname, 'eval-codebuff.json')],
-    agents: ['savant'],
-    taskIds: ['filter-system-history'],
-    saveTraces,
-  })
-
-  process.exit(0)
-}
-
-if (import.meta.main) {
-  main().catch((error) => {
-    console.error('Error running benchmark:', error)
-    process.exit(1)
-  })
-}

package/evals/benchmark/main.ts

@@ -1,29 +0,0 @@
-import path from 'path'
-
-import { runBenchmark } from './run-benchmark'
-
-async function main() {
-  const saveTraces = process.argv.includes('--save-traces')
-
-  // Compare SavantCode agents against external CLI agents
-  // Use 'external:claude' for Claude Code CLI
-  // Use 'external:codex' for OpenAI Codex CLI
-  // Use 'external:opencode' for OpenCode CLI
-  // NOTE: the default profile is the codebuff fixture (see main-hard-tasks.ts
-  // for the hard profile: eval-{codebuff,manifold,plane,saleor}2.json).
-  await runBenchmark({
-    evalDataPaths: [path.join(__dirname, 'eval-codebuff.json')],
-    agents: ['savant-free-evals'],
-    taskConcurrency: 6,
-    saveTraces,
-  })
-
-  process.exit(0)
-}
-
-if (import.meta.main) {
-  main().catch((error) => {
-    console.error('Error running example:', error)
-    process.exit(1)
-  })
-}

package/evals/benchmark/meta-analyzer.ts

@@ -1,329 +0,0 @@
-import fs from 'fs'
-import path from 'path'
-
-import { getErrorObject } from '@savant-code/common/util/error'
-
-import type { SavantCodeClient, AgentDefinition } from '@savant-code/sdk'
-
-export interface TaskAnalysisData {
-  commitSha: string
-  prompt: string
-  timestamp: string
-  overallAnalysis: string
-  agentFeedback: Array<{
-    agentId: string
-    strengths: string[]
-    weaknesses: string[]
-    recommendations: string[]
-  }>
-  results: Array<{
-    agentId: string
-    analysis: string
-    strengths: string[]
-    weaknesses: string[]
-    completionScore: number
-    codeQualityScore: number
-    overallScore: number
-    cost: number
-    durationMs: number
-    error?: string
-  }>
-}
-
-export interface MetaAnalysisResult {
-  overallComparison: string
-  agentInsights: Array<{
-    agentId: string
-    consistentStrengths: string[]
-    consistentWeaknesses: string[]
-    performanceSummary: string
-    recommendations: string[]
-  }>
-  keyFindings: string[]
-}
-
-const metaAnalyzerAgent: AgentDefinition = {
-  id: 'benchmark-meta-analyzer',
-  displayName: 'Benchmark Meta Analyzer',
-  model: 'openai/gpt-5',
-  toolNames: ['set_output'],
-  inputSchema: {
-    prompt: { type: 'string', description: 'The meta-analysis prompt' },
-  },
-  outputMode: 'structured_output',
-  outputSchema: {
-    type: 'object',
-    properties: {
-      overallComparison: {
-        type: 'string',
-        description: 'High-level comparison of all agents across all tasks',
-      },
-      agentInsights: {
-        type: 'array',
-        items: {
-          type: 'object',
-          properties: {
-            agentId: { type: 'string' },
-            consistentStrengths: {
-              type: 'array',
-              items: { type: 'string' },
-              description: 'Patterns of strengths across multiple tasks',
-            },
-            consistentWeaknesses: {
-              type: 'array',
-              items: { type: 'string' },
-              description: 'Patterns of weaknesses across multiple tasks',
-            },
-            performanceSummary: {
-              type: 'string',
-              description:
-                'Summary of overall performance including scores, cost, and time',
-            },
-            recommendations: {
-              type: 'array',
-              items: { type: 'string' },
-              description:
-                'High-level recommendations for improving this agent based on patterns observed',
-            },
-          },
-          required: [
-            'agentId',
-            'consistentStrengths',
-            'consistentWeaknesses',
-            'performanceSummary',
-            'recommendations',
-          ],
-        },
-      },
-      keyFindings: {
-        type: 'array',
-        items: { type: 'string' },
-        description:
-          'Most important insights from the evaluation that should guide development priorities',
-      },
-    },
-    required: ['overallComparison', 'agentInsights', 'keyFindings'],
-  },
-  systemPrompt: `You are an expert AI system evaluator analyzing patterns across multiple coding tasks and agents.
-
-## Your Role
-
-You will receive:
-1. Complete agent definitions showing their configuration, tools, prompts, and capabilities
-2. Agent type definitions explaining the available options and structure
-3. Trace analyses from multiple tasks showing how agents approached different problems
-4. Judge analyses showing the quality of their implementations
-5. Performance metrics (scores, costs, times) across all tasks
-
-## Focus on Patterns and Trends
-
-Your analysis should identify consistent patterns across multiple tasks:
-
-Key Analysis Areas:
-- **Agent Design Impact**: How does each agent's configuration (tools, model, prompts) affect their behavior and performance?
-- **Consistent Behaviors**: What patterns emerge in how each agent approaches problems?
-- **Performance Trends**: Which agents consistently score higher/lower? Why?
-- **Cost vs Quality Trade-offs**: How do agents balance thoroughness with efficiency?
-- **Reliability**: Which agents are more consistent vs variable in their performance?
-- **Comparative Analysis**: What are the key differentiators between agents? How do their configurations lead to different outcomes?
-- **Prompt Engineering Effectiveness**: Which agents have better-designed prompts that guide behavior effectively?
-
-## Output Format
-
-Provide:
-- **Overall Comparison**: High-level assessment comparing all agents' general approaches and performance
-- **Agent Insights**: For each agent:
-  - Consistent Strengths: Patterns that work well across multiple tasks
-  - Consistent Weaknesses: Recurring issues or limitations
-  - Performance Summary: Overall scores, costs, times, and reliability
-  - Recommendations: What changes would most improve this agent?
-- **Key Findings**: 3-5 most actionable insights that should guide development priorities
-
-Focus on actionable patterns that can inform agent improvements, not individual task details.`,
-}
-
-export async function analyzeAllTasks(params: {
-  client: SavantCodeClient
-  logsDir: string
-  agents: string[]
-  analyzerContext: {
-    agentDefinitions: AgentDefinition[]
-    agentTypeDefinition: string
-    testedAgentIds: string[]
-  }
-}): Promise<MetaAnalysisResult> {
-  const { client, logsDir, agents, analyzerContext } = params
-
-  try {
-    // Read all ANALYSIS files from logs directory
-    const files = fs.readdirSync(logsDir)
-    const analysisFiles = files.filter((f) => f.includes('ANALYSIS'))
-
-    const allTaskAnalyses: TaskAnalysisData[] = []
-    for (const file of analysisFiles) {
-      const filePath = path.join(logsDir, file)
-      const content = fs.readFileSync(filePath, 'utf-8')
-      const data: TaskAnalysisData = JSON.parse(content)
-      allTaskAnalyses.push(data)
-    }
-
-    if (allTaskAnalyses.length === 0) {
-      console.warn('No analysis files found in logs directory')
-      return {
-        overallComparison: 'No analysis data available',
-        agentInsights: [],
-        keyFindings: [],
-      }
-    }
-
-    // Create a concise summary for each task (without full agent traces)
-    const taskSummaries = allTaskAnalyses.map((task) => ({
-      prompt: task.prompt,
-      traceAnalysis: {
-        overallAnalysis: task.overallAnalysis,
-        agentFeedback: task.agentFeedback,
-      },
-      judgeResults: task.results.map((r) => ({
-        agentId: r.agentId,
-        overallScore: r.overallScore,
-        completionScore: r.completionScore,
-        codeQualityScore: r.codeQualityScore,
-        cost: r.cost,
-        durationMs: r.durationMs,
-        strengths: r.strengths,
-        weaknesses: r.weaknesses,
-        error: r.error,
-      })),
-    }))
-
-    // Filter agent definitions to only include tested agents
-    const filteredAgentDefinitions = analyzerContext.agentDefinitions.filter(
-      (def) => analyzerContext.testedAgentIds.includes(def.id),
-    )
-
-    const prompt = `## Agent Definitions Being Evaluated
-
-Below are the complete agent definitions for the agents being tested. Use this to understand their configuration, tools, prompts, and overall design.
-
-${JSON.stringify(filteredAgentDefinitions, null, 2)}
-
-## Agent Type Definition Reference
-
-For reference, here is the TypeScript type definition that agents use:
-
-\`\`\`typescript
-${analyzerContext.agentTypeDefinition}
-\`\`\`
-
-## All Task Analyses
-
-You are analyzing ${allTaskAnalyses.length} tasks evaluated across ${agents.length} agent(s): ${agents.join(', ')}
-
-${JSON.stringify(taskSummaries, null, 2)}
-
-Analyze these results to identify:
-
-1. **Overall Comparison**: How do the agents compare in general? What are the key differentiators?
-
-2. **Per-Agent Patterns**: For each agent, identify:
-   - What strengths appear consistently across tasks?
-   - What weaknesses or issues recur?
-   - How does their performance (scores, cost, time) compare?
-   - What patterns emerge in how they approach problems?
-
-3. **Actionable Insights**: What are the 3-5 most important findings that should guide development?
-   - Which improvements would have the biggest impact?
-   - What trade-offs are agents making?
-   - Are there reliability concerns?
-
-Focus on patterns across multiple tasks, not individual task details.`
-
-    const agentOutput: string[] = []
-    const analyzerResult = await client.run({
-      agent: 'benchmark-meta-analyzer',
-      prompt,
-      agentDefinitions: [metaAnalyzerAgent],
-      // FID-2026-0803-007 EV-3: abort the underlying LLM run on timeout.
-      signal: AbortSignal.timeout(30 * 60 * 1000),
-      handleEvent: (event) => {
-        if (event.type === 'text') {
-          agentOutput.push(event.text)
-        } else if (event.type === 'tool_call') {
-          agentOutput.push(JSON.stringify(event, null, 2))
-        } else if (event.type === 'error') {
-          console.warn('[Meta Analyzer] Error event:', event.message)
-        }
-      },
-    })
-
-    const { output } = analyzerResult
-
-    if (output.type !== 'structuredOutput' || output.value === null) {
-      console.error(
-        'Error running meta analyzer - not structured output',
-        JSON.stringify(output, null, 2),
-      )
-      console.error('Meta analyzer output trace:', agentOutput.join(''))
-      return {
-        overallComparison:
-          'Error running meta analyzer - not structured output',
-        agentInsights: [],
-        keyFindings: [],
-      }
-    }
-
-    // FID-2026-0803-007 EV-9: the downstream code defensively validates every
-    // field (typeof/Array.isArray guards + string filters below), so this cast
-    // feeds a manual validation path rather than trusting the value — kept as
-    // the documented intentional pattern rather than adding a zod schema.
-    const value = output.value as unknown as Record<string, unknown>
-    const rawInsights = Array.isArray(value.agentInsights)
-      ? value.agentInsights
-      : []
-    const agentInsights = rawInsights
-      .filter(
-        (item): item is Record<string, unknown> =>
-          typeof item === 'object' && item !== null && !Array.isArray(item),
-      )
-      .map((item) => ({
-        agentId: typeof item.agentId === 'string' ? item.agentId : '',
-        consistentStrengths: Array.isArray(item.consistentStrengths)
-          ? item.consistentStrengths.filter(
-              (s): s is string => typeof s === 'string',
-            )
-          : [],
-        consistentWeaknesses: Array.isArray(item.consistentWeaknesses)
-          ? item.consistentWeaknesses.filter(
-              (s): s is string => typeof s === 'string',
-            )
-          : [],
-        performanceSummary:
-          typeof item.performanceSummary === 'string'
-            ? item.performanceSummary
-            : '',
-        recommendations: Array.isArray(item.recommendations)
-          ? item.recommendations.filter(
-              (s): s is string => typeof s === 'string',
-            )
-          : [],
-      }))
-    const keyFindings = Array.isArray(value.keyFindings)
-      ? value.keyFindings.filter((s): s is string => typeof s === 'string')
-      : []
-    return {
-      overallComparison:
-        typeof value.overallComparison === 'string'
-          ? value.overallComparison
-          : '',
-      agentInsights,
-      keyFindings,
-    }
-  } catch (error) {
-    console.error(`Failed to analyze all tasks:`, getErrorObject(error))
-    return {
-      overallComparison: `Error running meta analyzer: ${getErrorObject(error).message}`,
-      agentInsights: [],
-      keyFindings: [],
-    }
-  }
-}