zidane 9.1.1 → 9.1.3

package/scripts/eval.ts

@@ -1,179 +0,0 @@
-#!/usr/bin/env bun
-/**
- * Eval runner CLI. Parses flags into ZIDANE_EVAL_* env vars and spawns the
- * real-inference test file. Keeps the flag surface stable regardless of how
- * `bun test` treats unknown argv.
- *
- * Usage:
- *   bun run test:eval --model claude-opus-4-8 --scorer claude-opus-4-8
- *   bun run test:eval --provider openai --model gpt-5 --filter vite --repeat 3
- *   bun run test:eval --provider all                       # every provider with an API key in env/.env
- *   bun run test:eval --provider baseten --models zai-org/GLM-5.1,moonshotai/Kimi-K2.6
- *   bun run test:eval --setups build,codemode -g vite-todolist-react --repeat 3   # Build vs Codemode
- *
- * Flags:
- *   --provider <name>        agent provider (anthropic|openai|openrouter|cerebras|arcee|baseten|local).
- *                            `*` or `all` runs every provider with a configured API key, on default models.
- *   --model <id>             agent model override
- *   --models <id1>,<id2>     run every eval on each listed model (requires a specific --provider)
- *   --judge-provider <name>  judge provider (defaults to agent provider)
- *   --judge-model | --scorer judge model override
- *   --filter | -g <text>     run only evals whose suite/id/tag contains <text>
- *   --repeat <n>             repeat each eval n times
- *   --concurrency | --jobs <n>  max eval cases to run in parallel (default 4)
- *   --report-dir <dir>       write per-case + run-summary JSON here
- *   --artifact-dir <dir>     write per-case result/events/transcript/workspace here
- *   --baseline [path]        write a settings+measurements snapshot after the run
- *                            (default path: benchmarks/evals/baseline.json)
- *   --extensions <names>     enable repo extensions for the run, comma-separated
- *                            (e.g. pxpipe,headroom — see benchmarks/evals/_extensions.ts)
- *   --setups <names>         run every eval once per agent setup and compare them
- *                            (e.g. build,codemode — see benchmarks/evals/_setups.ts)
- *   --timeout <ms>           per-test timeout (default 180000)
- */
-
-import { existsSync } from 'node:fs'
-import { resolve } from 'node:path'
-
-const DEFAULT_TIMEOUT_MS = '180000'
-const EVAL_TEST_FILE = 'test/eval-real.ts'
-const DEFAULT_BASELINE_PATH = 'benchmarks/evals/baseline.json'
-
-const FLAG_TO_ENV: Record<string, string> = {
-  'provider': 'ZIDANE_EVAL_PROVIDER',
-  'model': 'ZIDANE_EVAL_MODEL',
-  'models': 'ZIDANE_EVAL_MODELS',
-  'judge-provider': 'ZIDANE_EVAL_JUDGE_PROVIDER',
-  'judge-model': 'ZIDANE_EVAL_JUDGE_MODEL',
-  'scorer': 'ZIDANE_EVAL_JUDGE_MODEL',
-  'filter': 'ZIDANE_EVAL_FILTER',
-  'g': 'ZIDANE_EVAL_FILTER',
-  'repeat': 'ZIDANE_EVAL_REPEAT',
-  'concurrency': 'ZIDANE_EVAL_CONCURRENCY',
-  'jobs': 'ZIDANE_EVAL_CONCURRENCY',
-  'report-dir': 'ZIDANE_EVAL_REPORT_DIR',
-  'artifact-dir': 'ZIDANE_EVAL_ARTIFACT_DIR',
-  'baseline': 'ZIDANE_EVAL_BASELINE',
-  'extensions': 'ZIDANE_EVAL_EXTENSIONS',
-  'setups': 'ZIDANE_EVAL_SETUPS',
-}
-
-/** Flags whose value may be omitted, falling back to a default. */
-const OPTIONAL_VALUE_FLAGS: Record<string, string> = {
-  baseline: DEFAULT_BASELINE_PATH,
-}
-
-const NUMERIC_FLAGS = new Set(['repeat', 'concurrency', 'jobs', 'timeout'])
-
-export interface EvalCliParseOptions {
-  cwd?: string
-  now?: Date
-  existingEnv?: Record<string, string | undefined>
-}
-
-export interface EvalCliParseResult {
-  env: Record<string, string>
-  timeout: string
-  testFile: string
-}
-
-export class EvalCliError extends Error {
-  constructor(message: string, readonly exitCode = 2) {
-    super(message)
-    this.name = 'EvalCliError'
-  }
-}
-
-export function parseEvalCliArgs(argv: string[], options: EvalCliParseOptions = {}): EvalCliParseResult {
-  const env: Record<string, string> = {}
-  let timeout = DEFAULT_TIMEOUT_MS
-  for (let i = 0; i < argv.length; i++) {
-    const token = argv[i]
-    if (!token.startsWith('-'))
-      continue
-    const key = token.replace(/^-+/, '')
-    const eq = key.indexOf('=')
-    const rawKey = eq === -1 ? key : key.slice(0, eq)
-    const inlineVal = eq === -1 ? undefined : key.slice(eq + 1)
-    let value = inlineVal ?? (argv[i + 1] && !argv[i + 1].startsWith('-') ? argv[++i] : undefined)
-    if (!value) {
-      const fallback = OPTIONAL_VALUE_FLAGS[rawKey]
-      if (!fallback)
-        throw new EvalCliError(`Missing value for eval flag: --${rawKey}`)
-      value = fallback
-    }
-    if (NUMERIC_FLAGS.has(rawKey))
-      assertPositiveInteger(rawKey, value)
-    if (rawKey === 'timeout') {
-      timeout = value
-      continue
-    }
-    const envKey = FLAG_TO_ENV[rawKey]
-    if (!envKey)
-      throw new EvalCliError(`Unknown eval flag: --${rawKey}`)
-    env[envKey] = value
-  }
-
-  // Cross-flag validation. `_config.ts` re-checks at run time (env vars can
-  // come from elsewhere), but failing here gives a clean exit-2 before any
-  // test process spawns.
-  const provider = env.ZIDANE_EVAL_PROVIDER
-  const wildcard = provider === '*' || provider === 'all'
-  if (env.ZIDANE_EVAL_MODELS && env.ZIDANE_EVAL_MODEL)
-    throw new EvalCliError('Use either --model or --models, not both.')
-  if (env.ZIDANE_EVAL_MODELS && (!provider || wildcard))
-    throw new EvalCliError('--models requires a specific --provider.')
-  if (wildcard && env.ZIDANE_EVAL_MODEL)
-    throw new EvalCliError('--model cannot be combined with --provider=* — each detected provider runs its default model.')
-
-  const cwd = options.cwd ?? process.cwd()
-  const existingEnv = options.existingEnv ?? process.env
-  if (env.ZIDANE_EVAL_BASELINE)
-    env.ZIDANE_EVAL_BASELINE = resolve(cwd, env.ZIDANE_EVAL_BASELINE)
-  // Default a run-scoped artifact dir so the summary can render clickable links
-  // to each case's output. Overridable via --artifact-dir / ZIDANE_EVAL_ARTIFACT_DIR.
-  if (!env.ZIDANE_EVAL_ARTIFACT_DIR && !existingEnv.ZIDANE_EVAL_ARTIFACT_DIR) {
-    const stamp = (options.now ?? new Date()).toISOString().replace(/[:.]/g, '-')
-    env.ZIDANE_EVAL_ARTIFACT_DIR = `${cwd}/.zidane/evals/${stamp}`
-  }
-
-  return { env, timeout, testFile: resolve(cwd, EVAL_TEST_FILE) }
-}
-
-export async function runEvalCli(argv: string[]): Promise<number> {
-  let parsed: EvalCliParseResult
-  try {
-    parsed = parseEvalCliArgs(argv)
-  }
-  catch (err) {
-    if (err instanceof EvalCliError) {
-      console.error(err.message)
-      return err.exitCode
-    }
-    throw err
-  }
-
-  if (!existsSync(parsed.testFile)) {
-    console.error('`bun run test:eval` is only available from a zidane source checkout.')
-    console.error(`Missing ${parsed.testFile}.`)
-    return 1
-  }
-
-  const proc = Bun.spawn(['bun', 'test', `--timeout=${parsed.timeout}`, parsed.testFile], {
-    env: { ...process.env, ...parsed.env },
-    stdout: 'inherit',
-    stderr: 'inherit',
-    stdin: 'inherit',
-  })
-
-  return await proc.exited
-}
-
-function assertPositiveInteger(flag: string, value: string): void {
-  const parsed = Number(value)
-  if (!Number.isInteger(parsed) || parsed <= 0)
-    throw new EvalCliError(`Eval flag --${flag} must be a positive integer, got ${JSON.stringify(value)}`)
-}
-
-if (import.meta.main)
-  process.exit(await runEvalCli(process.argv.slice(2)))