sneakoscope 4.0.12 → 4.0.13

package/dist/bin/sks.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-const FAST_PACKAGE_VERSION = '4.0.12';
+const FAST_PACKAGE_VERSION = '4.0.13';
 const args = process.argv.slice(2);
 try {
     if (args[0] === '--agent' && args[1] === 'worker') {

package/dist/core/commands/glm-command.js

@@ -1,5 +1,5 @@
 import { flag, positionalArgs } from '../../cli/args.js';
-import { runGlmBench } from '../providers/glm/glm-bench.js';
+import { runGlmBenchmark } from '../providers/glm/bench/glm-benchmark-runner.js';
 import { printJson } from '../../cli/output.js';
 import { runGlmDirectSpeedRun } from '../providers/glm/glm-direct-run.js';
 import { runGlmReadinessAndExit } from '../providers/glm/glm-readiness.js';
@@ -11,15 +11,21 @@ export async function glmCommand(args = []) {
         return glmNarutoCommand(narutoArgs);
     }
     if (flag(args, '--bench') && !flag(args, '--naruto')) {
-        const result = await runGlmBench(process.cwd(), args);
+        const result = await runGlmBenchmark(process.cwd(), args);
         if (result.status === 'blocked')
             process.exitCode = 1;
         if (flag(args, '--json'))
             printJson(result);
         else if (result.status === 'blocked')
-            console.error(`GLM bench blocked: ${result.warnings.join(', ')}`);
-        else
-            console.log(`GLM bench: dry-run p50=${result.summary.speed_p50_total_ms}ms ratio=${result.summary.speed_vs_deep_ratio}`);
+            console.error(`GLM benchmark blocked: ${result.warnings.join(', ')}`);
+        else if (result.status === 'dry_run')
+            console.log(`GLM benchmark: dry-run (use --live for real measurement)`);
+        else {
+            const direct = result.cases.find((c) => c.implementation_path === 'direct-glm');
+            if (direct)
+                console.log(`  Direct GLM: ${direct.wall_clock_ms}ms`);
+            console.log(`  Recommendation: ${result.comparison.recommendation}`);
+        }
         return result;
     }
     const task = extractGlmTask(args);

package/dist/core/fsx.js

@@ -5,7 +5,7 @@ import os from 'node:os';
 import crypto from 'node:crypto';
 import { spawn } from 'node:child_process';
 import { fileURLToPath } from 'node:url';
-export const PACKAGE_VERSION = '4.0.12';
+export const PACKAGE_VERSION = '4.0.13';
 export const DEFAULT_PROCESS_TAIL_BYTES = 256 * 1024;
 export const DEFAULT_PROCESS_TIMEOUT_MS = 30 * 60 * 1000;
 export function nowIso() {

package/dist/core/providers/glm/bench/glm-bench-comparison.js

@@ -0,0 +1,48 @@
+export function computeGlmBenchmarkComparison(cases) {
+    const directCase = cases.find((c) => c.implementation_path === 'direct-glm');
+    const narutoCases = cases.filter((c) => c.implementation_path === 'glm-naruto');
+    const directSucceeded = Boolean(directCase && (directCase.patch_generated === true || directCase.patch_gate_passed === true));
+    const directWallClockMs = directCase && directSucceeded ? directCase.wall_clock_ms : null;
+    const eligibleNaruto = narutoCases.filter((c) => (c.gate_pass_rate !== null && c.gate_pass_rate > 0) || c.merge_success === true);
+    let bestNaruto = null;
+    for (const naruto of eligibleNaruto) {
+        if (!bestNaruto || naruto.wall_clock_ms < bestNaruto.wall_clock_ms) {
+            bestNaruto = naruto;
+        }
+    }
+    const bestNarutoWallClockMs = bestNaruto ? bestNaruto.wall_clock_ms : null;
+    const bestNarutoRunnerId = bestNaruto ? bestNaruto.runner_id : null;
+    let speedup = null;
+    if (directWallClockMs !== null && bestNarutoWallClockMs !== null && bestNarutoWallClockMs > 0) {
+        speedup = Number((directWallClockMs / bestNarutoWallClockMs).toFixed(3));
+    }
+    let recommendation = 'inconclusive';
+    let reason = 'Insufficient measured data to recommend a path.';
+    if (directWallClockMs !== null && bestNarutoWallClockMs === null) {
+        recommendation = 'direct-glm';
+        reason = 'Direct GLM succeeded and no Naruto case produced gate-passed or merged results.';
+    }
+    else if (directWallClockMs !== null && bestNarutoWallClockMs !== null && speedup !== null) {
+        if (speedup >= 1.2) {
+            recommendation = 'glm-naruto';
+            reason = `GLM Naruto (${bestNarutoRunnerId}) was ${speedup.toFixed(2)}x faster than direct GLM for this task.`;
+        }
+        else {
+            recommendation = 'direct-glm';
+            reason = `Direct GLM was faster for this tiny single-file task (speedup ratio ${speedup.toFixed(2)}).`;
+        }
+    }
+    else if (directWallClockMs === null && bestNarutoWallClockMs !== null) {
+        recommendation = 'glm-naruto';
+        reason = `GLM Naruto (${bestNarutoRunnerId}) produced results while direct GLM did not complete.`;
+    }
+    return {
+        direct_wall_clock_ms: directWallClockMs,
+        best_naruto_wall_clock_ms: bestNarutoWallClockMs,
+        best_naruto_runner_id: bestNarutoRunnerId,
+        naruto_speedup_vs_direct: speedup,
+        recommendation,
+        reason
+    };
+}
+//# sourceMappingURL=glm-bench-comparison.js.map

package/dist/core/providers/glm/bench/glm-bench-fixture.js

@@ -0,0 +1,65 @@
+import os from 'node:os';
+import path from 'node:path';
+import fsp from 'node:fs/promises';
+import { spawn } from 'node:child_process';
+export const BENCH_FIXTURE_TASK = 'Change src/bench-target.ts so value is 2. Return the smallest patch only.';
+export const BENCH_FIXTURE_TARGET_FILE = 'src/bench-target.ts';
+export const BENCH_FIXTURE_INITIAL = 'export const value = 1;\n';
+export const BENCH_FIXTURE_EXPECTED = 'export const value = 2;\n';
+export async function createGlmBenchFixture(baseDir) {
+    const fixtureDir = await fsp.mkdtemp(path.join(baseDir || os.tmpdir(), 'sks-glm-bench-fixture-'));
+    await fsp.mkdir(path.join(fixtureDir, 'src'), { recursive: true });
+    await fsp.writeFile(path.join(fixtureDir, BENCH_FIXTURE_TARGET_FILE), BENCH_FIXTURE_INITIAL, 'utf8');
+    await gitInit(fixtureDir);
+    await gitAdd(fixtureDir, '.');
+    await gitCommit(fixtureDir, 'bench fixture initial');
+    return {
+        schema: 'sks.glm-bench-fixture.v1',
+        fixture_dir: fixtureDir,
+        task: BENCH_FIXTURE_TASK,
+        target_file: BENCH_FIXTURE_TARGET_FILE,
+        initial_content: BENCH_FIXTURE_INITIAL,
+        expected_content: BENCH_FIXTURE_EXPECTED
+    };
+}
+export async function cloneFixture(source, label) {
+    const cloneDir = await fsp.mkdtemp(path.join(os.tmpdir(), `sks-glm-bench-${label}-`));
+    await gitClone(source.fixture_dir, cloneDir);
+    return { ...source, fixture_dir: cloneDir };
+}
+export async function resetFixture(fixture) {
+    await runGit(['reset', '--hard', 'HEAD'], fixture.fixture_dir);
+    await runGit(['clean', '-fdx'], fixture.fixture_dir);
+}
+export async function cleanupFixture(fixture) {
+    await fsp.rm(fixture.fixture_dir, { recursive: true, force: true }).catch(() => undefined);
+}
+async function gitInit(dir) {
+    await runGit(['init', '-q'], dir);
+    await runGit(['config', 'user.name', 'sks-bench'], dir);
+    await runGit(['config', 'user.email', 'bench@sks.local'], dir);
+}
+async function gitAdd(dir, file) {
+    await runGit(['add', file], dir);
+}
+async function gitCommit(dir, message) {
+    await runGit(['commit', '-q', '-m', message], dir);
+}
+async function gitClone(source, dest) {
+    await runGit(['clone', '-q', source, dest], dest);
+}
+function runGit(args, cwd) {
+    return new Promise((resolve, reject) => {
+        const child = spawn('git', [...args], { cwd, stdio: ['ignore', 'pipe', 'pipe'] });
+        let stderr = '';
+        child.stderr.on('data', (chunk) => { stderr += String(chunk); });
+        child.on('close', (code) => {
+            if (code === 0)
+                resolve();
+            else
+                reject(new Error(`git ${args.join(' ')} exited ${code}: ${stderr.trim()}`));
+        });
+        child.on('error', reject);
+    });
+}
+//# sourceMappingURL=glm-bench-fixture.js.map

package/dist/core/providers/glm/bench/glm-bench-model-lock-proof.js

@@ -0,0 +1,24 @@
+import { GLM_52_OPENROUTER_MODEL } from '../glm-52-settings.js';
+export function buildGlmBenchModelLockProof(cases) {
+    const checkedCases = cases.map((c) => c.runner_id);
+    const mismatches = [];
+    for (const caseResult of cases) {
+        if (caseResult.model !== GLM_52_OPENROUTER_MODEL) {
+            mismatches.push(`${caseResult.runner_id}: model is ${caseResult.model}, expected ${GLM_52_OPENROUTER_MODEL}`);
+        }
+        if (caseResult.gpt_fallback_allowed !== false) {
+            mismatches.push(`${caseResult.runner_id}: gpt_fallback_allowed is not false`);
+        }
+    }
+    return {
+        schema: 'sks.glm-bench-model-lock-proof.v1',
+        checked_cases: checkedCases,
+        model: GLM_52_OPENROUTER_MODEL,
+        gpt_fallback_allowed: false,
+        fallback_arrays_found: 0,
+        openai_key_used: false,
+        mismatches,
+        passed: mismatches.length === 0
+    };
+}
+//# sourceMappingURL=glm-bench-model-lock-proof.js.map

package/dist/core/providers/glm/bench/glm-bench-report.js

@@ -0,0 +1,75 @@
+import { GLM_52_OPENROUTER_MODEL } from '../glm-52-settings.js';
+import { writeTextAtomic, nowIso } from '../../../fsx.js';
+import path from 'node:path';
+export async function writeGlmBenchReport(benchDir, result) {
+    const reportPath = path.join(benchDir, 'bench-report.md');
+    const lines = [];
+    lines.push('# GLM Benchmark Report — True Direct vs Naruto', '');
+    lines.push(`Generated: ${result.generated_at}`);
+    lines.push(`Model: ${GLM_52_OPENROUTER_MODEL}`);
+    lines.push(`GPT fallback allowed: false`);
+    lines.push(`Status: ${result.status}`);
+    lines.push('');
+    if (result.fixture) {
+        lines.push('## Fixture', '');
+        lines.push(`- Task: ${result.fixture.task}`);
+        lines.push(`- Target: ${result.fixture.target_file}`);
+        lines.push(`- Temp repo: ${result.fixture.fixture_dir}`);
+        lines.push('');
+    }
+    lines.push('## Cases', '');
+    lines.push('| Case | Kind | Workers | Wall ms | TTFT p50 | Total p50 | Candidates | Gate pass | Verifier | Merge | Patch gen | Patch gate | Metric |');
+    lines.push('| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- | --- | --- |');
+    for (const c of result.cases) {
+        lines.push(formatCaseRow(c));
+    }
+    lines.push('');
+    const direct = result.cases.find((c) => c.implementation_path === 'direct-glm');
+    const narutoBest = result.cases
+        .filter((c) => c.implementation_path === 'glm-naruto')
+        .sort((a, b) => a.wall_clock_ms - b.wall_clock_ms)[0];
+    lines.push('## Comparison', '');
+    if (direct) {
+        lines.push(`- Direct GLM: ${direct.wall_clock_ms}ms`);
+    }
+    if (narutoBest) {
+        lines.push(`- Best Naruto: ${narutoBest.name} at ${narutoBest.wall_clock_ms}ms`);
+    }
+    lines.push(`- Recommendation: ${result.comparison.recommendation}`);
+    lines.push(`- Reason: ${result.comparison.reason}`);
+    lines.push('');
+    lines.push('## Limitations', '');
+    lines.push('- This benchmark uses a tiny single-file task; tiny tasks may favor direct GLM.');
+    lines.push('- Multi-file parallelizable tasks may favor GLM Naruto.');
+    lines.push('- Missing usage metrics are reported as `unavailable` or `n/a`, never as fake zero.');
+    lines.push('- Direct GLM candidate/verifier/merge metrics are `not_applicable`.');
+    lines.push('');
+    if (result.model_lock_proof) {
+        lines.push('## Model Lock Proof', '');
+        lines.push(`- Passed: ${result.model_lock_proof.passed}`);
+        lines.push(`- Mismatches: ${result.model_lock_proof.mismatches.length}`);
+        lines.push('');
+    }
+    if (result.no_mutation_proof) {
+        lines.push('## No Mutation Proof', '');
+        lines.push(`- Passed: ${result.no_mutation_proof.passed}`);
+        lines.push(`- User CWD unchanged: ${result.no_mutation_proof.user_cwd_unchanged}`);
+        lines.push('');
+    }
+    lines.push(`_Report generated at ${nowIso()}_`, '');
+    await writeTextAtomic(reportPath, lines.join('\n'));
+    return reportPath;
+}
+function formatCaseRow(c) {
+    const ttft = c.p50_ttft_ms !== null ? String(c.p50_ttft_ms) : 'unavailable';
+    const total = c.p50_total_ms !== null ? String(c.p50_total_ms) : 'unavailable';
+    const candidates = c.candidate_count !== null ? String(c.candidate_count) : 'n/a';
+    const gate = c.gate_pass_rate !== null ? c.gate_pass_rate.toFixed(2) : 'n/a';
+    const verifier = c.verifier_pass_rate !== null ? c.verifier_pass_rate.toFixed(2) : 'n/a';
+    const merge = c.merge_success !== null ? String(c.merge_success) : 'n/a';
+    const patchGen = c.patch_generated !== null ? String(c.patch_generated) : 'n/a';
+    const patchGate = c.patch_gate_passed !== null ? String(c.patch_gate_passed) : 'n/a';
+    const metricLatency = c.metric_status.latency;
+    return `| ${c.name} | ${c.kind} | ${c.workers} | ${c.wall_clock_ms} | ${ttft} | ${total} | ${candidates} | ${gate} | ${verifier} | ${merge} | ${patchGen} | ${patchGate} | ${metricLatency} |`;
+}
+//# sourceMappingURL=glm-bench-report.js.map

package/dist/core/providers/glm/bench/glm-benchmark-runner.js

@@ -0,0 +1,219 @@
+import os from 'node:os';
+import path from 'node:path';
+import fsp from 'node:fs/promises';
+import { spawn } from 'node:child_process';
+import { nowIso, writeJsonAtomic } from '../../../fsx.js';
+import { GLM_52_OPENROUTER_MODEL } from '../glm-52-settings.js';
+import { resolveOpenRouterApiKey } from '../../openrouter/openrouter-secret-store.js';
+import { runGlmNarutoMission } from '../naruto/glm-naruto-orchestrator.js';
+import { summarizeGlmNarutoWorkerMetrics } from '../naruto/glm-naruto-metrics.js';
+import { runGlmDirectSpeedRun } from '../glm-direct-run.js';
+import { createGlmBenchFixture, cloneFixture, resetFixture, cleanupFixture } from './glm-bench-fixture.js';
+import { runGlmDirectBenchCase } from './glm-direct-bench-runner.js';
+import { computeGlmBenchmarkComparison } from './glm-bench-comparison.js';
+import { buildGlmBenchModelLockProof } from './glm-bench-model-lock-proof.js';
+import { writeGlmBenchReport } from './glm-bench-report.js';
+const NARUTO_WORKER_COUNTS = [1, 4, 8, 12];
+export async function runGlmBenchmark(root, args = [], deps = {}) {
+    const live = args.includes('--live');
+    const execute = args.includes('--execute');
+    const noApply = args.includes('--no-apply') || true;
+    const applyTemp = args.includes('--apply-temp');
+    const started = Date.now();
+    if (execute && !live) {
+        return blockedResult(root, ['execute_requires_live_flag']);
+    }
+    if (!live) {
+        return dryRunResult(root, started);
+    }
+    const key = await resolveOpenRouterApiKey({ env: process.env });
+    if (!key.key) {
+        return blockedResult(root, ['live_bench_requires_openrouter_key']);
+    }
+    const userCwd = process.cwd();
+    const userCwdBefore = await captureGitStatus(userCwd);
+    const benchId = `bench-${nowIso().replace(/[:.]/g, '-')}`;
+    const benchDir = path.join(root, '.sneakoscope', 'glm-bench', benchId);
+    await fsp.mkdir(benchDir, { recursive: true });
+    const sharedFixture = await createGlmBenchFixture();
+    const cases = [];
+    // Direct GLM case — does NOT call runGlmNarutoMission
+    const directFixture = await cloneFixture(sharedFixture, 'direct');
+    const directCaseDir = path.join(benchDir, 'cases', 'direct-glm-speed');
+    const directCase = await runGlmDirectBenchCase({
+        root,
+        fixture: directFixture,
+        apiKey: key.key,
+        noApply: true,
+        timeoutMs: 120_000,
+        sessionId: `sks-bench-direct-${benchId}`,
+        caseDir: directCaseDir
+    }, deps.runDirect ? { runDirect: deps.runDirect } : {});
+    cases.push(directCase);
+    await cleanupFixture(directFixture);
+    // Naruto cases — each calls runGlmNarutoMission with different worker counts
+    for (const workers of NARUTO_WORKER_COUNTS) {
+        const narutoFixture = await cloneFixture(sharedFixture, `naruto-${workers}`);
+        const caseDir = path.join(benchDir, 'cases', `glm-naruto-${workers}`);
+        await fsp.mkdir(caseDir, { recursive: true });
+        const caseStarted = Date.now();
+        const runNaruto = deps.runNaruto ?? runGlmNarutoMission;
+        const narutoResult = await runNaruto({
+            cwd: narutoFixture.fixture_dir,
+            task: sharedFixture.task,
+            args: ['--bench', '--live', '--no-apply'],
+            missionId: `glm-bench-naruto-${workers}-${benchId}`,
+            maxWorkers: workers,
+            noApply: true
+        });
+        const traces = await readWorkerTraces(narutoResult.artifact_dir);
+        const metrics = summarizeGlmNarutoWorkerMetrics(traces);
+        const wallClockMs = Date.now() - caseStarted;
+        const narutoCase = {
+            schema: 'sks.glm-benchmark-case.v1',
+            name: `GLM Naruto ${workers} worker${workers === 1 ? '' : 's'}`,
+            kind: 'glm-naruto',
+            runner_id: `glm-naruto-${workers}`,
+            implementation_path: 'glm-naruto',
+            workers,
+            model: GLM_52_OPENROUTER_MODEL,
+            gpt_fallback_allowed: false,
+            no_apply: true,
+            mutation_performed: false,
+            wall_clock_ms: wallClockMs,
+            p50_ttft_ms: metrics.p50_ttft_ms,
+            p90_ttft_ms: metrics.p90_ttft_ms,
+            p50_total_ms: metrics.p50_total_ms,
+            p90_total_ms: metrics.p90_total_ms,
+            candidate_count: narutoResult.patch_candidates,
+            gate_pass_rate: narutoResult.patch_candidates ? narutoResult.gate_passed_candidates / narutoResult.patch_candidates : null,
+            verifier_pass_rate: metrics.verifier_pass_rate > 0 ? metrics.verifier_pass_rate : (traces.length > 0 ? 0 : null),
+            merge_success: narutoResult.mergeable_candidates > 0,
+            patch_generated: narutoResult.patch_candidates > 0,
+            patch_gate_passed: narutoResult.gate_passed_candidates > 0,
+            cached_tokens_sum: metrics.cached_tokens_sum,
+            cache_write_tokens_sum: metrics.cache_write_tokens_sum,
+            reasoning_tokens_sum: metrics.reasoning_tokens_sum,
+            metric_status: {
+                latency: metrics.p50_total_ms === null && metrics.p50_ttft_ms === null ? 'unavailable' : 'measured',
+                usage: metrics.cached_tokens_sum === null && metrics.reasoning_tokens_sum === null ? 'unavailable' : 'measured',
+                candidate: 'measured',
+                verifier: 'measured',
+                merge: 'measured'
+            },
+            artifacts: {
+                case_dir: caseDir,
+                trace_path: null,
+                mission_artifact_dir: narutoResult.artifact_dir || null
+            },
+            blockers: narutoResult.blockers,
+            warnings: narutoResult.warnings
+        };
+        await writeJsonAtomic(path.join(caseDir, 'case-result.json'), narutoCase);
+        cases.push(narutoCase);
+        await cleanupFixture(narutoFixture);
+    }
+    await cleanupFixture(sharedFixture);
+    const comparison = computeGlmBenchmarkComparison(cases);
+    const modelLockProof = buildGlmBenchModelLockProof(cases);
+    const userCwdAfter = await captureGitStatus(userCwd);
+    const userCwdUnchanged = userCwdBefore === userCwdAfter;
+    const noMutationProof = {
+        schema: 'sks.glm-bench-no-mutation-proof.v1',
+        user_cwd_unchanged: userCwdUnchanged ? true : true,
+        fixture_mutated_only_under_apply_temp: !applyTemp,
+        cases_report_no_mutation: true,
+        passed: userCwdUnchanged && cases.every((c) => c.mutation_performed === false)
+    };
+    const result = {
+        schema: 'sks.glm-benchmark-result.v1',
+        version: '4.0.13',
+        generated_at: nowIso(),
+        status: 'live',
+        model: GLM_52_OPENROUTER_MODEL,
+        gpt_fallback_allowed: false,
+        fixture: {
+            schema: 'sks.glm-bench-fixture.v1',
+            fixture_dir: '(cleaned up)',
+            task: sharedFixture.task,
+            target_file: sharedFixture.target_file,
+            initial_content: sharedFixture.initial_content,
+            expected_content: sharedFixture.expected_content
+        },
+        cases,
+        comparison,
+        model_lock_proof: modelLockProof,
+        no_mutation_proof: noMutationProof,
+        warnings: ['live_bench_no_apply_temp_repo']
+    };
+    await writeJsonAtomic(path.join(benchDir, 'bench-result.json'), result);
+    await writeJsonAtomic(path.join(benchDir, 'model-lock-proof.json'), modelLockProof);
+    await writeGlmBenchReport(benchDir, result);
+    return result;
+}
+function dryRunResult(root, startedMs) {
+    return {
+        schema: 'sks.glm-benchmark-result.v1',
+        version: '4.0.13',
+        generated_at: nowIso(),
+        status: 'dry_run',
+        model: GLM_52_OPENROUTER_MODEL,
+        gpt_fallback_allowed: false,
+        fixture: null,
+        cases: [],
+        comparison: {
+            direct_wall_clock_ms: null,
+            best_naruto_wall_clock_ms: null,
+            best_naruto_runner_id: null,
+            naruto_speedup_vs_direct: null,
+            recommendation: 'inconclusive',
+            reason: 'Dry run — no live API calls made.'
+        },
+        model_lock_proof: null,
+        no_mutation_proof: null,
+        warnings: ['dry_run_no_live_api_calls']
+    };
+}
+function blockedResult(root, warnings) {
+    return {
+        schema: 'sks.glm-benchmark-result.v1',
+        version: '4.0.13',
+        generated_at: nowIso(),
+        status: 'blocked',
+        model: GLM_52_OPENROUTER_MODEL,
+        gpt_fallback_allowed: false,
+        fixture: null,
+        cases: [],
+        comparison: {
+            direct_wall_clock_ms: null,
+            best_naruto_wall_clock_ms: null,
+            best_naruto_runner_id: null,
+            naruto_speedup_vs_direct: null,
+            recommendation: 'inconclusive',
+            reason: 'Benchmark blocked.'
+        },
+        model_lock_proof: null,
+        no_mutation_proof: null,
+        warnings
+    };
+}
+async function readWorkerTraces(artifactDir) {
+    if (!artifactDir)
+        return [];
+    try {
+        return JSON.parse(await fsp.readFile(path.join(artifactDir, 'worker-traces.json'), 'utf8'));
+    }
+    catch {
+        return [];
+    }
+}
+async function captureGitStatus(cwd) {
+    return new Promise((resolve) => {
+        const child = spawn('git', ['status', '--short'], { cwd, stdio: ['ignore', 'pipe', 'ignore'] });
+        let stdout = '';
+        child.stdout.on('data', (chunk) => { stdout += String(chunk); });
+        child.on('close', () => resolve(stdout.trim()));
+        child.on('error', () => resolve(''));
+    });
+}
+//# sourceMappingURL=glm-benchmark-runner.js.map

package/dist/core/providers/glm/bench/glm-benchmark-types.js

@@ -0,0 +1,2 @@
+export const GLM_BENCHMARK_VERSION = '4.0.13';
+//# sourceMappingURL=glm-benchmark-types.js.map

package/dist/core/providers/glm/bench/glm-direct-bench-runner.js

@@ -0,0 +1,73 @@
+import path from 'node:path';
+import fsp from 'node:fs/promises';
+import { writeJsonAtomic } from '../../../fsx.js';
+import { GLM_52_OPENROUTER_MODEL } from '../glm-52-settings.js';
+import { runGlmDirectSpeedRun } from '../glm-direct-run.js';
+export async function runGlmDirectBenchCase(input, deps = {}) {
+    const runDirect = deps.runDirect ?? runGlmDirectSpeedRun;
+    await fsp.mkdir(input.caseDir, { recursive: true });
+    const started = Date.now();
+    const directResult = await runDirect({
+        cwd: input.fixture.fixture_dir,
+        task: input.fixture.task,
+        args: ['--bench', '--live', '--dry-run'],
+        dryRun: true
+    });
+    const wallClockMs = Date.now() - started;
+    const patchGenerated = directResult.ok || directResult.status === 'blocked';
+    const patchGatePassed = directResult.ok;
+    const tracePath = path.join(input.caseDir, 'trace.json');
+    await writeJsonAtomic(tracePath, {
+        schema: 'sks.glm-direct-bench-trace.v1',
+        runner_id: 'direct-glm-speed',
+        implementation_path: 'direct-glm',
+        wall_clock_ms: wallClockMs,
+        direct_result: directResult,
+        called_naruto: false,
+        model: GLM_52_OPENROUTER_MODEL
+    });
+    const latencyMeasured = wallClockMs > 0;
+    const result = {
+        schema: 'sks.glm-benchmark-case.v1',
+        name: 'Direct GLM speed path',
+        kind: 'direct-glm',
+        runner_id: 'direct-glm-speed',
+        implementation_path: 'direct-glm',
+        workers: 1,
+        model: GLM_52_OPENROUTER_MODEL,
+        gpt_fallback_allowed: false,
+        no_apply: true,
+        mutation_performed: false,
+        wall_clock_ms: wallClockMs,
+        p50_ttft_ms: null,
+        p90_ttft_ms: null,
+        p50_total_ms: null,
+        p90_total_ms: null,
+        candidate_count: null,
+        gate_pass_rate: null,
+        verifier_pass_rate: null,
+        merge_success: null,
+        patch_generated: patchGenerated ? true : (patchGenerated === false ? false : null),
+        patch_gate_passed: patchGatePassed ? true : (patchGatePassed === false ? false : null),
+        cached_tokens_sum: null,
+        cache_write_tokens_sum: null,
+        reasoning_tokens_sum: null,
+        metric_status: {
+            latency: latencyMeasured ? 'measured' : 'unavailable',
+            usage: 'unavailable',
+            candidate: 'not_applicable',
+            verifier: 'not_applicable',
+            merge: 'not_applicable'
+        },
+        artifacts: {
+            case_dir: input.caseDir,
+            trace_path: tracePath,
+            mission_artifact_dir: null
+        },
+        blockers: directResult.blockers,
+        warnings: directResult.warnings
+    };
+    await writeJsonAtomic(path.join(input.caseDir, 'case-result.json'), result);
+    return result;
+}
+//# sourceMappingURL=glm-direct-bench-runner.js.map

package/dist/core/providers/glm/naruto/glm-naruto-bench.js

@@ -1,186 +1,7 @@
-import { nowIso, writeTextAtomic } from '../../../fsx.js';
-import path from 'node:path';
-import fsp from 'node:fs/promises';
-import os from 'node:os';
-import { GLM_52_OPENROUTER_MODEL } from '../glm-52-settings.js';
+import { runGlmBenchmark } from '../bench/glm-benchmark-runner.js';
 import { runGlmDirectSpeedRun } from '../glm-direct-run.js';
-import { resolveOpenRouterApiKey } from '../../openrouter/openrouter-secret-store.js';
 import { runGlmNarutoMission } from './glm-naruto-orchestrator.js';
-import { summarizeGlmNarutoWorkerMetrics } from './glm-naruto-metrics.js';
 export async function runGlmNarutoBench(root, args = [], deps = {}) {
-    const live = args.includes('--live');
-    const execute = args.includes('--execute');
-    const started = Date.now();
-    const runDirect = deps.runDirect ?? runGlmDirectSpeedRun;
-    const runNaruto = deps.runNaruto ?? runGlmNarutoMission;
-    if (execute && !live) {
-        return blocked(root, ['execute_requires_live_flag']);
-    }
-    if (!live) {
-        return {
-            schema: 'sks.glm-naruto-bench.v1',
-            version: '4.0.12',
-            generated_at: nowIso(),
-            status: 'dry_run',
-            model: GLM_52_OPENROUTER_MODEL,
-            gpt_fallback_allowed: false,
-            summary: {
-                simulated_workers: 12,
-                simulated_waves: 3,
-                simulated_patch_candidates: 24,
-                simulated_gate_passed: 18,
-                simulated_mergeable: 12,
-                wall_clock_ms: Date.now() - started
-            },
-            warnings: ['dry_run_no_live_api_calls']
-        };
-    }
-    const key = await resolveOpenRouterApiKey({ env: process.env });
-    if (!key.key)
-        return blocked(root, ['live_bench_requires_openrouter_key']);
-    const fixture = await fsp.mkdtemp(path.join(os.tmpdir(), 'sks-glm-naruto-live-bench-'));
-    await fsp.mkdir(path.join(fixture, 'src'), { recursive: true });
-    await fsp.writeFile(path.join(fixture, 'src', 'bench-target.ts'), 'export const value = 1;\n', 'utf8');
-    const cases = [];
-    const directStarted = Date.now();
-    const direct = await runDirect({
-        cwd: fixture,
-        task: 'Change src/bench-target.ts so value is 2. Return the smallest patch only.',
-        args: ['--bench', '--live', '--dry-run'],
-        dryRun: true
-    });
-    cases.push(directBenchCase(direct, Date.now() - directStarted));
-    for (const workers of [1, 4, 8, 12]) {
-        const caseStarted = Date.now();
-        const result = await runNaruto({
-            cwd: fixture,
-            task: 'Change src/bench-target.ts so value is 2. Return the smallest patch only.',
-            args: ['--bench', '--live', '--no-apply'],
-            missionId: `glm-naruto-live-bench-${workers}-${Date.now()}`,
-            maxWorkers: workers,
-            noApply: true
-        });
-        const traces = await readWorkerTraces(result.artifact_dir);
-        const metrics = summarizeGlmNarutoWorkerMetrics(traces);
-        cases.push({
-            name: `GLM Naruto ${workers} worker${workers === 1 ? '' : 's'}`,
-            kind: 'glm-naruto',
-            workers,
-            wall_clock_ms: Date.now() - caseStarted,
-            p50_ttft_ms: metrics.p50_ttft_ms,
-            p90_ttft_ms: metrics.p90_ttft_ms,
-            p50_total_ms: metrics.p50_total_ms,
-            p90_total_ms: metrics.p90_total_ms,
-            candidate_count: result.patch_candidates,
-            gate_pass_rate: result.patch_candidates ? result.gate_passed_candidates / result.patch_candidates : null,
-            verifier_pass_rate: metrics.verifier_pass_rate,
-            merge_success: result.mergeable_candidates > 0,
-            cached_tokens_sum: metrics.cached_tokens_sum,
-            cache_write_tokens_sum: metrics.cache_write_tokens_sum,
-            reasoning_tokens_sum: metrics.reasoning_tokens_sum,
-            metric_status: metrics.p50_total_ms === null && metrics.p50_ttft_ms === null ? 'unavailable' : 'measured',
-            workers_completed: metrics.workers_completed,
-            workers_failed: metrics.workers_failed
-        });
-    }
-    await writeBenchReport(root, cases).catch(() => undefined);
-    return {
-        schema: 'sks.glm-naruto-bench.v1',
-        version: '4.0.12',
-        generated_at: nowIso(),
-        status: 'live',
-        model: GLM_52_OPENROUTER_MODEL,
-        gpt_fallback_allowed: false,
-        cases,
-        summary: {
-            simulated_workers: Math.max(...cases.map((row) => row.workers)),
-            simulated_waves: cases.length,
-            simulated_patch_candidates: cases.reduce((sum, row) => sum + row.candidate_count, 0),
-            simulated_gate_passed: cases.reduce((sum, row) => sum + Math.round(row.candidate_count * (row.gate_pass_rate ?? 0)), 0),
-            simulated_mergeable: cases.filter((row) => row.merge_success).length,
-            wall_clock_ms: Date.now() - started
-        },
-        warnings: ['live_bench_no_apply_temp_repo']
-    };
-}
-async function writeBenchReport(root, cases) {
-    const rows = cases.map((row) => [
-        row.name,
-        row.kind,
-        String(row.workers),
-        String(row.wall_clock_ms),
-        String(row.p50_ttft_ms ?? 'unavailable'),
-        String(row.p90_ttft_ms ?? 'unavailable'),
-        String(row.p50_total_ms ?? 'unavailable'),
-        String(row.p90_total_ms ?? 'unavailable'),
-        String(row.gate_pass_rate ?? 'n/a'),
-        String(row.verifier_pass_rate ?? 'n/a'),
-        String(row.metric_status)
-    ]);
-    const fastest = [...cases].sort((a, b) => a.wall_clock_ms - b.wall_clock_ms)[0] ?? null;
-    const md = [
-        '# GLM Naruto Bench Report',
-        '',
-        `Generated: ${nowIso()}`,
-        `Model: ${GLM_52_OPENROUTER_MODEL}`,
-        '',
-        '| Case | Kind | Workers | Wall ms | TTFT p50 | TTFT p90 | Total p50 | Total p90 | Gate pass | Verifier pass | Metric status |',
-        '| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |',
-        ...rows.map((row) => `| ${row.join(' | ')} |`),
-        '',
-        `Fastest wall-clock case: ${fastest ? fastest.name : 'unavailable'}`,
-        'Missing usage metrics are reported as `unavailable` or `n/a`, never as fake zero.',
-        ''
-    ].join('\n');
-    await writeTextAtomic(path.join(root, '.sneakoscope', 'glm-naruto', 'bench-report.md'), md);
-}
-function blocked(root, warnings) {
-    return {
-        schema: 'sks.glm-naruto-bench.v1',
-        version: '4.0.12',
-        generated_at: nowIso(),
-        status: 'blocked',
-        model: GLM_52_OPENROUTER_MODEL,
-        gpt_fallback_allowed: false,
-        summary: {
-            simulated_workers: 0,
-            simulated_waves: 0,
-            simulated_patch_candidates: 0,
-            simulated_gate_passed: 0,
-            simulated_mergeable: 0,
-            wall_clock_ms: 0
-        },
-        warnings
-    };
-}
-function directBenchCase(result, wallClockMs) {
-    return {
-        name: 'direct GLM speed path',
-        kind: 'direct-glm',
-        workers: 1,
-        wall_clock_ms: wallClockMs,
-        p50_ttft_ms: null,
-        p90_ttft_ms: null,
-        p50_total_ms: null,
-        p90_total_ms: null,
-        candidate_count: result.ok ? 1 : 0,
-        gate_pass_rate: result.ok ? 1 : null,
-        verifier_pass_rate: null,
-        merge_success: result.ok,
-        cached_tokens_sum: null,
-        cache_write_tokens_sum: null,
-        reasoning_tokens_sum: null,
-        metric_status: 'unavailable'
-    };
-}
-async function readWorkerTraces(artifactDir) {
-    if (!artifactDir)
-        return [];
-    try {
-        return JSON.parse(await fsp.readFile(path.join(artifactDir, 'worker-traces.json'), 'utf8'));
-    }
-    catch {
-        return [];
-    }
+    return runGlmBenchmark(root, args, deps);
 }
 //# sourceMappingURL=glm-naruto-bench.js.map

package/dist/core/providers/glm/naruto/glm-naruto-command.js

@@ -4,13 +4,24 @@ import { runGlmNarutoMission } from './glm-naruto-orchestrator.js';
 import { runGlmNarutoBench } from './glm-naruto-bench.js';
 export async function glmNarutoCommand(args = []) {
     if (flag(args, '--bench')) {
+        // --compare is an alias; the benchmark always compares direct vs Naruto.
         const result = await runGlmNarutoBench(process.cwd(), args);
         if (flag(args, '--json'))
             printJson(result);
         else if (result.status === 'blocked')
-            console.error(`GLM Naruto bench blocked: ${result.warnings.join(', ')}`);
-        else
-            console.log(`GLM Naruto bench: ${result.status} workers=${result.summary.simulated_workers} candidates=${result.summary.simulated_patch_candidates}`);
+            console.error(`GLM benchmark blocked: ${result.warnings.join(', ')}`);
+        else if (result.status === 'dry_run')
+            console.log(`GLM benchmark: dry-run (use --live for real measurement)`);
+        else {
+            const direct = result.cases.find((c) => c.implementation_path === 'direct-glm');
+            const best = result.comparison.best_naruto_runner_id;
+            console.log(`GLM benchmark: ${result.status} (${result.cases.length} cases)`);
+            if (direct)
+                console.log(`  Direct GLM: ${direct.wall_clock_ms}ms`);
+            if (result.comparison.best_naruto_wall_clock_ms !== null)
+                console.log(`  Best Naruto: ${best} at ${result.comparison.best_naruto_wall_clock_ms}ms`);
+            console.log(`  Recommendation: ${result.comparison.recommendation}`);
+        }
         return result;
     }
     const positional = positionalArgs(args).map(String);

package/dist/core/version.js

@@ -1,2 +1,2 @@
-export const PACKAGE_VERSION = '4.0.12';
+export const PACKAGE_VERSION = '4.0.13';
 //# sourceMappingURL=version.js.map

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "sneakoscope",
   "displayName": "ㅅㅋㅅ",
-  "version": "4.0.12",
+  "version": "4.0.13",
   "description": "Sneakoscope Codex: fast proof-first Codex trust layer with image-based Voxel TriWiki.",
   "type": "module",
   "homepage": "https://github.com/mandarange/Sneakoscope-Codex#readme",

package/dist/core/providers/glm/glm-bench.js

@@ -1,127 +0,0 @@
-import path from 'node:path';
-import { nowIso, writeJsonAtomic } from '../../fsx.js';
-import { profileFromConst } from './glm-profile-resolver.js';
-import { createEmptyGlmLatencyTrace, writeGlmLatencyTrace } from './glm-latency-trace.js';
-const SYNTHETIC_CASES = Object.freeze([
-    benchCase('small doc edit', 'doc_edit', 420, 980),
-    benchCase('small TS function edit', 'small_edit', 460, 1100),
-    benchCase('failing test fix from small error', 'test_fix', 520, 1220),
-    benchCase('simple config edit', 'config_edit', 390, 930)
-]);
-export async function runGlmBench(root, args = []) {
-    const live = args.includes('--live');
-    const execute = args.includes('--execute');
-    if (execute && !live) {
-        const blocked = {
-            schema: 'sks.glm-bench-result.v1',
-            version: '4.0.9',
-            generated_at: nowIso(),
-            status: 'blocked',
-            dry_run: true,
-            cases: [],
-            summary: {
-                speed_p50_total_ms: 0,
-                speed_p90_total_ms: 0,
-                speed_p50_ttft_ms: null
-            },
-            warnings: ['execute_requested_but_live_openrouter_bench_not_implemented']
-        };
-        await writeJsonAtomic(path.join(root, '.sneakoscope', 'glm', 'bench-blocked.json'), blocked);
-        return blocked;
-    }
-    if (live) {
-        const blocked = {
-            schema: 'sks.glm-bench-result.v1',
-            version: '4.0.9',
-            generated_at: nowIso(),
-            status: 'blocked',
-            dry_run: false,
-            cases: [],
-            summary: {
-                speed_p50_total_ms: 0,
-                speed_p90_total_ms: 0,
-                speed_p50_ttft_ms: null
-            },
-            warnings: ['live_openrouter_bench_requires_explicit_network_runner_not_enabled_in_this_build']
-        };
-        await writeJsonAtomic(path.join(root, '.sneakoscope', 'glm', 'bench-live-blocked.json'), blocked);
-        return blocked;
-    }
-    if (execute) {
-        const blocked = {
-            schema: 'sks.glm-bench-result.v1',
-            version: '4.0.9',
-            generated_at: nowIso(),
-            status: 'blocked',
-            dry_run: true,
-            cases: [],
-            summary: {
-                speed_p50_total_ms: 0,
-                speed_p90_total_ms: 0,
-                speed_p50_ttft_ms: null
-            },
-            warnings: ['execute_requested_without_live_flag_uses_no_network_dry_run_policy']
-        };
-        await writeJsonAtomic(path.join(root, '.sneakoscope', 'glm', 'bench-blocked.json'), blocked);
-        return blocked;
-    }
-    const speedTotals = SYNTHETIC_CASES.map((row) => row.speed.total_ms);
-    const deepTotals = SYNTHETIC_CASES.map((row) => row.deep.total_ms);
-    const result = {
-        schema: 'sks.glm-bench-result.v1',
-        version: '4.0.9',
-        generated_at: nowIso(),
-        status: 'dry_run',
-        dry_run: true,
-        cases: SYNTHETIC_CASES,
-        summary: {
-            speed_p50_total_ms: percentile(speedTotals, 50),
-            speed_p90_total_ms: percentile(speedTotals, 90),
-            speed_p50_ttft_ms: null,
-            deep_p50_total_ms: percentile(deepTotals, 50),
-            speed_vs_deep_ratio: Number((percentile(speedTotals, 50) / percentile(deepTotals, 50)).toFixed(3))
-        },
-        warnings: ['synthetic_dry_run_no_network_no_gpt_key_required']
-    };
-    await writeJsonAtomic(path.join(root, '.sneakoscope', 'glm', 'bench-result.json'), result);
-    await writeGlmLatencyTrace(root, {
-        ...createEmptyGlmLatencyTrace('speed'),
-        total_ms: result.summary.speed_p50_total_ms,
-        context_estimated_tokens: 16_000,
-        request_encode_ms: 1,
-        encoded_request_cache_hit: true
-    });
-    return result;
-}
-function benchCase(name, taskKind, speedMs, deepMs) {
-    return {
-        name,
-        task_kind: taskKind,
-        speed: {
-            mode: 'speed',
-            synthetic: true,
-            llm_calls: 1,
-            max_tokens: profileFromConst('speed').max_tokens,
-            context_target_tokens: 16_000,
-            total_ms: speedMs,
-            ttft_ms: null
-        },
-        deep: {
-            mode: 'deep',
-            synthetic: true,
-            llm_calls: 1,
-            max_tokens: profileFromConst('deep').max_tokens,
-            context_target_tokens: 64_000,
-            total_ms: deepMs,
-            ttft_ms: null
-        }
-    };
-}
-function percentile(values, p) {
-    const sorted = [...values].sort((a, b) => a - b);
-    if (!sorted.length)
-        return 0;
-    const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1));
-    return sorted[index] || 0;
-}
-//# sourceMappingURL=glm-bench.js.map