sneakoscope 4.8.7 → 5.1.2

package/dist/core/verification/machine-feedback.js

@@ -0,0 +1,146 @@
+import path from 'node:path';
+import { exists, listFilesRecursive, readJson, runProcess } from '../fsx.js';
+import { runCompiledRules } from './mistake-rule-compiler.js';
+export async function runMachineFeedback(root, changedFiles, opts = {}) {
+    const t0 = Date.now();
+    const timeoutMs = Math.max(5_000, opts.timeoutMs ?? 60_000);
+    const [typecheck, lint, tests] = await Promise.all([
+        runTypecheck(root, changedFiles, timeoutMs),
+        runLint(root, changedFiles, timeoutMs),
+        runSelectedTests(root, changedFiles, timeoutMs)
+    ]);
+    return {
+        schema: 'sks.machine-feedback.v1',
+        ok: typecheck.ok && lint.ok && tests.ok,
+        typecheck,
+        lint,
+        tests,
+        duration_ms: Date.now() - t0
+    };
+}
+export async function selectTests(root, changedFiles) {
+    const out = new Set();
+    for (const file of changedFiles.map(normalizePath).filter(Boolean)) {
+        const base = path.basename(file).replace(/\.(?:[cm]?[jt]sx?)$/, '');
+        for (const cand of await globTests(root, base))
+            out.add(cand);
+        for (const cand of await testsImporting(root, file))
+            out.add(cand);
+    }
+    return [...out].sort().slice(0, 20);
+}
+async function runTypecheck(root, changedFiles, timeoutMs) {
+    if (!changedFiles.some((file) => /\.(?:tsx?|mts|cts)$/.test(file)))
+        return { ok: true, errors: [], skipped_reason: 'no_ts_files_changed' };
+    const pkg = await packageJson(root);
+    const script = scriptNamed(pkg, 'typecheck');
+    const tsconfig = await exists(path.join(root, 'tsconfig.json'));
+    if (!script && !tsconfig)
+        return { ok: true, errors: [], skipped_reason: 'tool_not_found' };
+    const command = script ? ['npm', ['run', 'typecheck', '--silent']] : ['npx', ['tsc', '--noEmit', '-p', 'tsconfig.json']];
+    const result = await runProcess(command[0], command[1], { cwd: root, timeoutMs, maxOutputBytes: 512 * 1024 });
+    if (result.timedOut)
+        return { ok: true, errors: [], skipped_reason: 'timeout' };
+    return {
+        ok: result.code === 0,
+        errors: result.code === 0 ? [] : summarizeErrors(result.stderr || result.stdout)
+    };
+}
+async function runLint(root, changedFiles, timeoutMs) {
+    const rules = await runCompiledRules(root, changedFiles).catch((err) => ({
+        ok: false,
+        violations: [{ rule_id: 'rule-runner', severity: 'error', file: '', line: 0, description: err instanceof Error ? err.message : String(err), good_example: '' }],
+        rule_count: 0
+    }));
+    const ruleErrors = rules.violations.map((violation) => `${violation.file}${violation.line ? `:${violation.line}` : ''} ${violation.rule_id}: ${violation.description}${violation.good_example ? ` | good: ${singleLine(violation.good_example)}` : ''}`);
+    const pkg = await packageJson(root);
+    const lintScript = scriptNamed(pkg, 'lint');
+    if (!lintScript) {
+        return {
+            ok: rules.ok,
+            errors: ruleErrors.slice(0, 20),
+            ...(rules.rule_count ? {} : { skipped_reason: 'skipped_reason' in rules ? rules.skipped_reason || 'tool_not_found' : 'tool_not_found' })
+        };
+    }
+    const args = ['run', 'lint', '--silent'];
+    const result = await runProcess('npm', args, { cwd: root, timeoutMs, maxOutputBytes: 512 * 1024 });
+    if (result.timedOut)
+        return { ok: rules.ok, errors: ruleErrors.slice(0, 20), skipped_reason: 'timeout' };
+    return {
+        ok: result.code === 0 && rules.ok,
+        errors: [...(result.code === 0 ? [] : summarizeErrors(result.stderr || result.stdout)), ...ruleErrors].slice(0, 20)
+    };
+}
+async function runSelectedTests(root, changedFiles, timeoutMs) {
+    const selected = await selectTests(root, changedFiles);
+    if (!selected.length)
+        return { ok: true, selected: [], failed: [], skipped_reason: 'no_related_tests' };
+    const failed = [];
+    const runnable = selected.filter((file) => /\.(?:mjs|cjs|js)$/.test(file)).slice(0, 10);
+    if (!runnable.length) {
+        const pkg = await packageJson(root);
+        if (!scriptNamed(pkg, 'test'))
+            return { ok: true, selected, failed: [], skipped_reason: 'no_directly_runnable_tests' };
+        const result = await runProcess('npm', ['test', '--silent'], { cwd: root, timeoutMs, maxOutputBytes: 512 * 1024 });
+        if (result.timedOut)
+            return { ok: true, selected, failed: [], skipped_reason: 'timeout' };
+        return { ok: result.code === 0, selected, failed: result.code === 0 ? [] : summarizeErrors(result.stderr || result.stdout).slice(0, 10) };
+    }
+    await Promise.all(runnable.map(async (file) => {
+        const result = await runProcess(process.execPath, [file], { cwd: root, timeoutMs: Math.max(5_000, Math.floor(timeoutMs / runnable.length)), maxOutputBytes: 256 * 1024 });
+        if (result.timedOut)
+            return;
+        if (result.code !== 0)
+            failed.push(`${file}: ${summarizeErrors(result.stderr || result.stdout).join(' | ')}`);
+    }));
+    return { ok: failed.length === 0, selected, failed: failed.slice(0, 10) };
+}
+async function globTests(root, base) {
+    const files = await listFilesRecursive(root, { ignore: ['.git', 'node_modules', 'dist', '.sneakoscope/tmp'], maxFiles: 30_000 });
+    return files
+        .map((file) => normalizePath(path.relative(root, file)))
+        .filter((file) => isTestFile(file))
+        .filter((file) => {
+        const name = path.basename(file);
+        return name.startsWith(`${base}.test.`)
+            || name.startsWith(`${base}.spec.`)
+            || file.includes(`/__tests__/${base}`);
+    });
+}
+async function testsImporting(root, changedFile) {
+    const files = await listFilesRecursive(root, { ignore: ['.git', 'node_modules', 'dist', '.sneakoscope/tmp'], maxFiles: 30_000 });
+    const stem = normalizePath(changedFile).replace(/\.(?:[cm]?[jt]sx?)$/, '');
+    const out = [];
+    await Promise.all(files.map(async (abs) => {
+        const rel = normalizePath(path.relative(root, abs));
+        if (!isTestFile(rel))
+            return;
+        const text = await import('node:fs/promises').then((fs) => fs.readFile(abs, 'utf8')).catch(() => '');
+        if (String(text).includes(changedFile) || String(text).includes(stem) || String(text).includes(`../${stem}`))
+            out.push(rel);
+    }));
+    return out;
+}
+function isTestFile(file) {
+    return /(?:^|\/)__tests__\//.test(file) || /\.(?:test|spec)\.(?:[cm]?[jt]sx?)$/.test(file);
+}
+async function packageJson(root) {
+    return readJson(path.join(root, 'package.json'), {}).catch(() => ({}));
+}
+function scriptNamed(pkg, name) {
+    const script = pkg?.scripts?.[name];
+    return typeof script === 'string' && script.trim() ? script : null;
+}
+function summarizeErrors(text) {
+    return String(text || '').split(/\r?\n/)
+        .map((line) => line.trim())
+        .filter(Boolean)
+        .slice(0, 20);
+}
+function normalizePath(value) {
+    return String(value || '').replace(/\\/g, '/').replace(/^\.\/+/, '');
+}
+function singleLine(value) {
+    return String(value || '').replace(/\s+/g, ' ').trim().slice(0, 240);
+}
+//# sourceMappingURL=machine-feedback.js.map

package/dist/core/verification/mistake-rule-compiler.js

@@ -0,0 +1,195 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { ensureDir, listFilesRecursive, nowIso, readJson, readText, runProcess, which, writeJsonAtomic } from '../fsx.js';
+import { readWrongnessLedger, writeWrongnessLedger } from '../triwiki-wrongness/wrongness-ledger.js';
+export async function compileMistakeRules(root) {
+    const ledger = await readWrongnessLedger(root);
+    const compiled = [];
+    const skipped = [];
+    const records = Array.isArray(ledger.records) ? ledger.records : [];
+    let changed = false;
+    for (const record of records) {
+        if (record.rule_compiled === true)
+            continue;
+        const rule = synthesizeRule(record);
+        if (rule && await validateRule(root, rule)) {
+            await writeRule(root, rule);
+            record.rule_compiled = true;
+            record.compiled_rule_id = rule.id;
+            record.updated_at = nowIso();
+            compiled.push(rule);
+            changed = true;
+        }
+        else {
+            skipped.push(String(record.id || 'unknown'));
+        }
+    }
+    if (changed)
+        await writeWrongnessLedger(root, { ...ledger, records });
+    await writeJsonAtomic(path.join(root, '.sneakoscope', 'rules', 'compiled-index.json'), {
+        schema: 'sks.mistake-rule-index.v1',
+        generated_at: nowIso(),
+        compiled: compiled.map((rule) => rule.id),
+        skipped
+    });
+    return { compiled, skipped };
+}
+export async function runCompiledRules(root, changedFiles) {
+    const rules = await readCompiledRules(root);
+    if (!rules.length)
+        return { ok: true, violations: [], rule_count: 0, skipped_reason: 'no_rules' };
+    const files = changedFiles.length ? changedFiles : (await listFilesRecursive(root, { ignore: ['.git', 'node_modules', 'dist'], maxFiles: 2000 })).map((file) => path.relative(root, file));
+    const astGrep = await which('ast-grep');
+    const violations = [];
+    for (const rule of rules) {
+        if (rule.detector.kind === 'ast-grep') {
+            if (!astGrep)
+                continue;
+            violations.push(...await runAstGrepRule(root, rule, files));
+        }
+        else {
+            violations.push(...await runRegexRule(root, rule, files));
+        }
+    }
+    return {
+        ok: !violations.some((violation) => violation.severity === 'error'),
+        violations,
+        rule_count: rules.length
+    };
+}
+async function readCompiledRules(root) {
+    const dir = path.join(root, '.sneakoscope', 'rules');
+    const entries = await fs.readdir(dir, { withFileTypes: true }).catch(() => []);
+    const rules = [];
+    for (const entry of entries) {
+        if (!entry.isFile() || !entry.name.endsWith('.rule.json'))
+            continue;
+        const rule = await readJson(path.join(dir, entry.name), null);
+        if (rule?.schema === 'sks.mistake-rule.v1')
+            rules.push(rule);
+    }
+    return rules;
+}
+function synthesizeRule(record) {
+    const id = `wl-${String(record?.id || '').replace(/[^A-Za-z0-9_-]+/g, '-').slice(0, 96)}`;
+    const text = [
+        record?.avoidance_rule?.text,
+        record?.claim?.text,
+        record?.corrective_action?.summary,
+        record?.detected_by?.detail
+    ].map((part) => String(part || '')).join('\n');
+    const bad = String(record?.examples?.bad || record?.example_bad || record?.bad || '');
+    const good = String(record?.examples?.good || record?.example_good || record?.good || '');
+    const explicitPattern = String(record?.detector?.pattern || record?.regex || record?.forbidden_pattern || '');
+    const nPlusOne = /n\+1|query.*loop|loop.*query|쿼리.*반복/i.test(text);
+    const emptyCatch = /empty\s+catch|catch\s*\{\s*\}|빈\s*catch/i.test(text);
+    const pattern = explicitPattern
+        || (nPlusOne ? String.raw `for\s*\([^)]*\)\s*\{[\s\S]{0,500}\b(?:query|findMany|findUnique|select|fetch)\s*\(` : '')
+        || (emptyCatch ? String.raw `catch\s*\([^)]*\)\s*\{\s*\}` : '');
+    if (!pattern)
+        return null;
+    return {
+        schema: 'sks.mistake-rule.v1',
+        id,
+        source: 'wrongness-ledger',
+        description: String(record?.avoidance_rule?.text || record?.claim?.text || record?.wrongness_kind || 'Compiled wrongness rule').slice(0, 500),
+        detector: { kind: 'regex', pattern, file_glob: String(record?.detector?.file_glob || '**/*.{ts,tsx,js,mjs,cjs}') },
+        severity: record?.severity === 'low' ? 'warn' : 'error',
+        examples: {
+            bad: bad || exampleForPattern(pattern, true),
+            good: good || exampleForPattern(pattern, false)
+        }
+    };
+}
+async function validateRule(root, rule) {
+    if (rule.detector.kind === 'ast-grep') {
+        if (!(await which('ast-grep')))
+            return false;
+        const tmp = path.join(root, '.sneakoscope', 'tmp', 'rule-validate', rule.id);
+        await ensureDir(tmp);
+        const bad = path.join(tmp, `bad.${rule.detector.lang || 'ts'}`);
+        const good = path.join(tmp, `good.${rule.detector.lang || 'ts'}`);
+        await fs.writeFile(bad, rule.examples.bad, 'utf8');
+        await fs.writeFile(good, rule.examples.good, 'utf8');
+        const badRun = await runProcess('ast-grep', ['run', '-p', rule.detector.pattern, bad], { cwd: root, timeoutMs: 10_000, maxOutputBytes: 64 * 1024 });
+        const goodRun = await runProcess('ast-grep', ['run', '-p', rule.detector.pattern, good], { cwd: root, timeoutMs: 10_000, maxOutputBytes: 64 * 1024 });
+        return Boolean(badRun.stdout.trim()) && !goodRun.stdout.trim();
+    }
+    try {
+        const re = new RegExp(rule.detector.pattern, 'm');
+        return re.test(rule.examples.bad) && !re.test(rule.examples.good);
+    }
+    catch {
+        return false;
+    }
+}
+async function writeRule(root, rule) {
+    await writeJsonAtomic(path.join(root, '.sneakoscope', 'rules', `${rule.id}.rule.json`), rule);
+}
+async function runRegexRule(root, rule, files) {
+    if (rule.detector.kind !== 'regex')
+        return [];
+    const detector = rule.detector;
+    let re;
+    try {
+        re = new RegExp(detector.pattern, 'gm');
+    }
+    catch {
+        return [];
+    }
+    const out = [];
+    for (const file of files.filter((candidate) => fileGlobMatches(candidate, detector.file_glob))) {
+        const text = await readText(path.join(root, file), '').catch(() => '');
+        re.lastIndex = 0;
+        for (const match of String(text).matchAll(re)) {
+            out.push(ruleViolation(rule, file, lineForIndex(String(text), match.index || 0)));
+            if (out.length >= 100)
+                return out;
+        }
+    }
+    return out;
+}
+async function runAstGrepRule(root, rule, files) {
+    const out = [];
+    for (const file of files) {
+        const run = await runProcess('ast-grep', ['run', '-p', rule.detector.pattern, path.join(root, file)], {
+            cwd: root,
+            timeoutMs: 10_000,
+            maxOutputBytes: 64 * 1024
+        }).catch(() => null);
+        if (!run?.stdout.trim())
+            continue;
+        out.push(ruleViolation(rule, file, 1));
+    }
+    return out;
+}
+function ruleViolation(rule, file, line) {
+    return {
+        rule_id: rule.id,
+        severity: rule.severity,
+        file,
+        line,
+        description: rule.description,
+        good_example: rule.examples.good
+    };
+}
+function fileGlobMatches(file, glob) {
+    const normalized = file.replace(/\\/g, '/');
+    if (glob.includes('{ts,tsx,js,mjs,cjs}'))
+        return /\.(?:tsx?|mjs|cjs|js)$/.test(normalized);
+    if (glob === '**/*')
+        return true;
+    const suffix = glob.replace(/^\*\*\/*/, '').replace(/^\*/, '');
+    return suffix ? normalized.endsWith(suffix) : true;
+}
+function lineForIndex(text, index) {
+    return text.slice(0, index).split(/\r?\n/).length;
+}
+function exampleForPattern(pattern, bad) {
+    if (/catch/.test(pattern))
+        return bad ? 'try { run(); } catch (err) {}' : 'try { run(); } catch (err) { throw err; }';
+    return bad
+        ? 'for (const id of ids) { await db.user.findUnique({ where: { id } }); }'
+        : 'await db.user.findMany({ where: { id: { in: ids } } });';
+}
+//# sourceMappingURL=mistake-rule-compiler.js.map

package/dist/core/version.js

@@ -1,2 +1,2 @@
-export const PACKAGE_VERSION = '4.8.7';
+export const PACKAGE_VERSION = '5.1.2';
 //# sourceMappingURL=version.js.map

package/dist/core/zellij/zellij-self-heal.js

@@ -1,6 +1,7 @@
 import fs from 'node:fs/promises';
 import path from 'node:path';
 import { ensureDir, nowIso, runProcess, writeJsonAtomic } from '../fsx.js';
+import { withHeartbeat } from '../../cli/cli-theme.js';
 import { createRequestedScopeContract } from '../safety/requested-scope-contract.js';
 import { guardContextForRoute, guardedPackageInstall } from '../safety/mutation-guard.js';
 import { mutationLedgerPath } from '../safety/mutation-ledger.js';
@@ -80,7 +81,7 @@ export async function repairZellijForSks(input) {
                 : manualResult(root, input, env, before, latest, brew, policy.blockers[0] || 'homebrew_missing');
         }
         homebrewInstallAttempted = true;
-        const homebrewRun = await runHomebrewInstall(root, env);
+        const homebrewRun = await runHomebrewInstall(root, env, input.quiet === true);
         if (homebrewRun.code !== 0) {
             return persistSelfHeal(root, input.missionDir, {
                 schema: 'sks.zellij-self-heal.v1',
@@ -112,7 +113,7 @@ export async function repairZellijForSks(input) {
         : install ? 'brew-install-zellij'
             : 'brew-upgrade-zellij';
     const command = `brew ${brewArgs.join(' ')}`;
-    const run = await runZellijBrew(root, env, brewBin, brewArgs, command);
+    const run = await runZellijBrew(root, env, brewBin, brewArgs, command, input.quiet === true);
     if (run.code !== 0 && /already installed|already up-to-date/i.test(`${run.stdout}\n${run.stderr}`)) {
         const after = await capabilitySnapshot(root, env, 'after-noop');
         return persistSelfHeal(root, input.missionDir, {
@@ -250,7 +251,7 @@ async function findBrew(env) {
         return { present: true, bin: 'brew' };
     return { present: false, bin: null };
 }
-async function runZellijBrew(root, env, brewBin, args, command) {
+async function runZellijBrew(root, env, brewBin, args, command, quiet = false) {
     if (env.SKS_ZELLIJ_SELF_HEAL_FAKE_RUN === '1') {
         await appendFakeBrewLog(env, args);
         return { code: Number(env.SKS_ZELLIJ_SELF_HEAL_FAKE_RUN_CODE || 0), stdout: 'fake brew ok', stderr: '' };
@@ -261,7 +262,7 @@ async function runZellijBrew(root, env, brewBin, args, command) {
         projectRoot: root,
         overrides: { package_install: true, zellij_install: true }
     });
-    return guardedPackageInstall(guardContextForRoute(root, contract, command), 'zellij', {
+    const work = guardedPackageInstall(guardContextForRoute(root, contract, command), 'zellij', {
         confirmed: true,
         command: brewBin,
         args,
@@ -269,8 +270,9 @@ async function runZellijBrew(root, env, brewBin, args, command) {
         timeoutMs: 180000,
         maxOutputBytes: 256 * 1024
     }).catch((err) => ({ code: 1, stdout: '', stderr: errorMessage(err) }));
+    return quiet ? await work : await withHeartbeat('brew zellij repair', work, { warnAfterMs: 30_000 });
 }
-async function runHomebrewInstall(root, env) {
+async function runHomebrewInstall(root, env, quiet = false) {
     if (env.SKS_ZELLIJ_SELF_HEAL_FAKE_RUN === '1') {
         await appendFakeBrewLog(env, ['install-homebrew']);
         return { code: Number(env.SKS_ZELLIJ_SELF_HEAL_FAKE_HOMEBREW_CODE || 0), stdout: 'fake homebrew install ok', stderr: '' };
@@ -281,7 +283,7 @@ async function runHomebrewInstall(root, env) {
         projectRoot: root,
         overrides: { package_install: true, zellij_install: true }
     });
-    return guardedPackageInstall(guardContextForRoute(root, contract, HOMEBREW_INSTALL_COMMAND), 'homebrew', {
+    const work = guardedPackageInstall(guardContextForRoute(root, contract, HOMEBREW_INSTALL_COMMAND), 'homebrew', {
         confirmed: true,
         command: '/bin/bash',
         args: ['-c', 'curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh | /bin/bash'],
@@ -289,6 +291,7 @@ async function runHomebrewInstall(root, env) {
         timeoutMs: 600000,
         maxOutputBytes: 256 * 1024
     }).catch((err) => ({ code: 1, stdout: '', stderr: errorMessage(err) }));
+    return quiet ? await work : await withHeartbeat('Homebrew install', work, { warnAfterMs: 60_000 });
 }
 async function appendFakeBrewLog(env, args) {
     if (!env.SKS_FAKE_BREW_LOG)

package/dist/scripts/check-feature-quality.js

@@ -40,8 +40,8 @@ function releaseQualityBlockers(counts = {}) {
     const blockers = [];
     if (Number(counts.runtime_verified || 0) < 22)
         blockers.push(`runtime_verified_below_release_target:${counts.runtime_verified || 0}<22`);
-    if (Number(counts.runtime_mock_verified || 0) < 45)
-        blockers.push(`runtime_mock_verified_below_release_target:${counts.runtime_mock_verified || 0}<45`);
+    if (Number(counts.wiring_only || 0) > 30)
+        blockers.push(`wiring_only_above_release_target:${counts.wiring_only || 0}>30`);
     if (Number(counts.integration_optional || 0) > 6)
         blockers.push(`integration_optional_above_release_target:${counts.integration_optional || 0}>6`);
     if (Number(counts.static_contract || 0) > 45)

package/dist/scripts/cli-output-consistency-check.js

@@ -0,0 +1,57 @@
+#!/usr/bin/env node
+import fs from 'node:fs';
+import { spawnSync } from 'node:child_process';
+import { assertGate, emitGate, root } from './sks-1-18-gate-lib.js';
+import { PACKAGE_VERSION } from '../core/version.js';
+const executed = [
+    ['ui', '--once'],
+    ['review'],
+    ['fast-mode', 'status'],
+    ['menubar', 'status'],
+    ['uninstall', '--dry-run', '--yes'],
+    ['naruto', 'help']
+];
+const staticFiles = [
+    'src/commands/doctor.ts',
+    'src/core/commands/basic-cli.ts',
+    'src/core/commands/naruto-command.ts',
+    'src/core/commands/review-command.ts',
+    'src/core/commands/ui-command.ts',
+    'src/core/commands/fast-mode-command.ts',
+    'src/core/commands/menubar-command.ts',
+    'src/core/commands/uninstall-command.ts'
+];
+const runs = executed.map((args) => run(args));
+const staticMissing = staticFiles.filter((file) => !fs.readFileSync(file, 'utf8').includes('.banner('));
+const badFirstLines = runs.filter((row) => !row.first_line.startsWith(`SKS ${PACKAGE_VERSION} ·`));
+const missingVocabulary = runs.filter((row) => !/[✔▲✖]/.test(row.stdout));
+const report = {
+    schema: 'sks.cli-output-consistency.v1',
+    ok: badFirstLines.length === 0 && missingVocabulary.length === 0 && staticMissing.length === 0,
+    executed: runs,
+    static_files_checked: staticFiles,
+    bad_first_lines: badFirstLines,
+    missing_status_vocabulary: missingVocabulary,
+    static_missing_banner: staticMissing
+};
+assertGate(report.ok, 'CLI output consistency check failed', report);
+emitGate('cli:output-consistency', report);
+function run(args) {
+    const result = spawnSync(process.execPath, ['./dist/bin/sks.js', ...args], {
+        cwd: root,
+        encoding: 'utf8',
+        env: { ...process.env, NO_COLOR: '1', SKS_UNINSTALL_SKIP_TMP_SWEEP: '1' },
+        timeout: 120000,
+        maxBuffer: 1024 * 1024
+    });
+    const stdout = String(result.stdout || '');
+    const stderr = String(result.stderr || '');
+    return {
+        args,
+        code: result.status,
+        first_line: stdout.split(/\r?\n/).find(Boolean) || '',
+        stdout: stdout.slice(0, 4000),
+        stderr: stderr.slice(0, 1000)
+    };
+}
+//# sourceMappingURL=cli-output-consistency-check.js.map

package/dist/scripts/coding-bench-check.js

@@ -0,0 +1,136 @@
+#!/usr/bin/env node
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { ensureDir, nowIso, readJson, runProcess, writeJsonAtomic } from '../core/fsx.js';
+const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..');
+const tasksRoot = path.join(root, 'bench', 'tasks');
+const reportPath = path.join(root, '.sneakoscope', 'reports', 'bench-report.json');
+const baselinePath = path.join(root, 'config', 'bench-baseline.json');
+const baseline = await readJson(baselinePath, { schema: 'sks.coding-bench-baseline.v1', pass_rate: 0, min_pass_rate: 0 });
+const tasks = await loadTasks();
+const backend = backendAvailable();
+const perTask = [];
+for (const task of tasks) {
+    if (!backend.ok) {
+        perTask.push({
+            id: task.id,
+            kind: task.kind,
+            ok: true,
+            status: 'skipped',
+            skipped_reason: backend.reason
+        });
+        continue;
+    }
+    perTask.push(await runTask(task));
+}
+const executed = perTask.filter((task) => task.status !== 'skipped');
+const pass = executed.filter((task) => task.ok).length;
+const fail = executed.filter((task) => !task.ok).length;
+const skipped = perTask.length - executed.length;
+const passRate = executed.length ? pass / executed.length : 0;
+const baselineRate = Number(baseline.pass_rate ?? baseline.min_pass_rate ?? 0);
+const ok = fail === 0 && passRate >= baselineRate;
+const report = {
+    schema: 'sks.coding-bench-report.v1',
+    generated_at: nowIso(),
+    ok,
+    pass,
+    fail,
+    skipped,
+    total: perTask.length,
+    pass_rate: passRate,
+    baseline: {
+        path: path.relative(root, baselinePath),
+        pass_rate: baselineRate,
+        baseline_version: baseline.baseline_version || null
+    },
+    backend,
+    per_task: perTask
+};
+await ensureDir(path.dirname(reportPath));
+await writeJsonAtomic(reportPath, report);
+console.log(JSON.stringify(report, null, 2));
+if (!ok)
+    process.exit(1);
+async function loadTasks() {
+    const dirs = (await fs.readdir(tasksRoot, { withFileTypes: true }).catch(() => []))
+        .filter((entry) => entry.isDirectory())
+        .map((entry) => entry.name)
+        .sort();
+    const rows = [];
+    for (const dir of dirs) {
+        const taskPath = path.join(tasksRoot, dir, 'task.json');
+        const task = await readJson(taskPath, null);
+        if (task?.schema === 'sks.bench-task.v1')
+            rows.push({ ...task, dir, task_path: taskPath, repo_path: path.join(tasksRoot, dir, 'repo') });
+    }
+    return rows;
+}
+function backendAvailable() {
+    if (process.env.SKS_RUN_CODING_BENCH_REAL !== '1')
+        return { ok: false, reason: 'no_backend' };
+    if (process.env.OPENAI_API_KEY || process.env.SKS_CODEX_LB_READY === '1' || process.env.SKS_CODING_BENCH_ALLOW_NO_KEY === '1')
+        return { ok: true, reason: null };
+    return { ok: false, reason: 'no_backend' };
+}
+async function runTask(task) {
+    const scratch = await fs.mkdtemp(path.join(os.tmpdir(), `sks-coding-bench-${task.id}-`));
+    await fs.cp(task.repo_path, scratch, { recursive: true });
+    await initGit(scratch);
+    const sks = path.join(root, 'dist', 'bin', 'sks.js');
+    const run = await runProcess(process.execPath, [sks, 'naruto', 'run', task.prompt, '--json', '--no-open-zellij', '--apply-patches', '--tournament', task.kind === 'refactor' ? '3' : '0'], {
+        cwd: scratch,
+        timeoutMs: 10 * 60_000,
+        maxOutputBytes: 512 * 1024,
+        env: { ...process.env, SKS_CODING_BENCH_TASK_ID: task.id }
+    });
+    const grade = await gradeTask(task, scratch);
+    return {
+        id: task.id,
+        kind: task.kind,
+        ok: run.code === 0 && grade.ok,
+        status: run.code === 0 && grade.ok ? 'passed' : 'failed',
+        command_exit: run.code,
+        grade,
+        scratch
+    };
+}
+async function initGit(cwd) {
+    await runProcess('git', ['init'], { cwd, timeoutMs: 15_000, maxOutputBytes: 64 * 1024 }).catch(() => null);
+    await runProcess('git', ['config', 'user.email', 'bench@sneakoscope.local'], { cwd, timeoutMs: 15_000, maxOutputBytes: 64 * 1024 }).catch(() => null);
+    await runProcess('git', ['config', 'user.name', 'Sneakoscope Bench'], { cwd, timeoutMs: 15_000, maxOutputBytes: 64 * 1024 }).catch(() => null);
+    await runProcess('git', ['add', '.'], { cwd, timeoutMs: 15_000, maxOutputBytes: 64 * 1024 }).catch(() => null);
+    await runProcess('git', ['commit', '-m', 'bench baseline'], { cwd, timeoutMs: 15_000, maxOutputBytes: 64 * 1024 }).catch(() => null);
+}
+async function gradeTask(task, cwd) {
+    const success = task.success || {};
+    const test = success.command ? await runProcess('sh', ['-lc', String(success.command)], { cwd, timeoutMs: 120_000, maxOutputBytes: 256 * 1024 }) : { code: 0, stdout: '', stderr: '' };
+    const diff = await runProcess('git', ['diff', '--numstat', 'HEAD'], { cwd, timeoutMs: 15_000, maxOutputBytes: 128 * 1024 }).catch(() => ({ stdout: '' }));
+    const changed = await runProcess('git', ['diff', '--name-only', 'HEAD'], { cwd, timeoutMs: 15_000, maxOutputBytes: 128 * 1024 }).catch(() => ({ stdout: '' }));
+    const patch = await runProcess('git', ['diff', 'HEAD'], { cwd, timeoutMs: 15_000, maxOutputBytes: 512 * 1024 }).catch(() => ({ stdout: '' }));
+    const diffLines = String(diff.stdout || '').split(/\r?\n/).filter(Boolean).reduce((sum, line) => {
+        const parts = line.split(/\s+/).map((value) => Number(value));
+        const added = parts[0] ?? 0;
+        const removed = parts[1] ?? 0;
+        return sum + (Number.isFinite(added) ? added : 0) + (Number.isFinite(removed) ? removed : 0);
+    }, 0);
+    const changedFiles = String(changed.stdout || '').split(/\r?\n/).filter(Boolean);
+    const forbidden = (success.forbidden_patterns || []).filter((pattern) => new RegExp(pattern, 'm').test(String(patch.stdout || '')));
+    const missingNewTest = success.must_contain_new_test === true && !changedFiles.some((file) => /\.(test|spec)\.[cm]?[jt]s$|test\.[cm]?js$|__tests__\//.test(file));
+    const tooLarge = Number(success.max_diff_lines || Infinity) < diffLines;
+    const ok = (success.must_pass !== true || test.code === 0) && !missingNewTest && !tooLarge && forbidden.length === 0;
+    return {
+        ok,
+        test_exit: test.code,
+        diff_lines: diffLines,
+        changed_files: changedFiles,
+        forbidden_patterns: forbidden,
+        missing_new_test: missingNewTest,
+        max_diff_lines_exceeded: tooLarge,
+        stdout_tail: String(test.stdout || '').slice(-4000),
+        stderr_tail: String(test.stderr || '').slice(-4000)
+    };
+}
+//# sourceMappingURL=coding-bench-check.js.map