swarm-orchestrator 7.0.0-alpha.0

package/dist/src/verifier/outcome-checks.js

@@ -0,0 +1,396 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runOutcomeChecks = runOutcomeChecks;
+const child_process_1 = require("child_process");
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const defaults_1 = require("../defaults");
+function last20Lines(output) {
+    const lines = output.split('\n');
+    if (lines.length <= 20)
+        return output;
+    return '...\n' + lines.slice(-20).join('\n');
+}
+function runOutcomeChecks(opts, workingDir) {
+    const checks = [];
+    checks.push(checkGitDiff(opts.workdir, opts.baseSha));
+    if (opts.expectedFiles && opts.expectedFiles.length > 0) {
+        checks.push(checkFileExistence(opts.workdir, opts.expectedFiles));
+    }
+    const buildCheck = checkBuildExec(opts.workdir);
+    if (buildCheck) {
+        checks.push(buildCheck);
+    }
+    if (process.env.SWARM_SKIP_OUTCOME_TEST_EXEC !== '1') {
+        const testCheck = checkTestExec(opts.workdir, workingDir);
+        if (testCheck) {
+            checks.push(testCheck);
+        }
+    }
+    // Idempotency resolution: if the diff check failed (no new commits) but the
+    // build and tests both pass, the goal state is demonstrably achieved. Downgrade
+    // git_diff from a hard failure to an advisory warning so an agent that found
+    // its work already present does not block the step from merging.
+    // Requires at least one exec check to be present; absence of scripts is not
+    // treated as a pass since that would mask genuinely empty agent runs.
+    const diffCheck = checks.find((c) => c.type === 'git_diff');
+    if (diffCheck && !diffCheck.passed) {
+        const buildExecCheck = checks.find((c) => c.type === 'build_exec');
+        const testExecCheck = checks.find((c) => c.type === 'test_exec');
+        const hasExecEvidence = buildExecCheck !== undefined || testExecCheck !== undefined;
+        const buildPassed = buildExecCheck ? buildExecCheck.passed : true;
+        const testsPassed = testExecCheck ? testExecCheck.passed : true;
+        if (hasExecEvidence && buildPassed && testsPassed) {
+            diffCheck.required = false;
+            diffCheck.passed = true;
+            diffCheck.evidence =
+                `${diffCheck.reason ?? 'No new commits'} — build and tests confirm goal already achieved (idempotent)`;
+            delete diffCheck.reason;
+        }
+    }
+    return checks;
+}
+function checkGitDiff(workdir, baseSha) {
+    const timeout = defaults_1.DEFAULT_SIGKILL_DELAY_MS * 2;
+    try {
+        // Primary: committed changes since baseline
+        const committedDiff = (0, child_process_1.execSync)(`git diff --stat ${baseSha}..HEAD`, { cwd: workdir, encoding: 'utf8', timeout }).trim();
+        if (committedDiff) {
+            const lines = committedDiff.split('\n');
+            const summaryLine = lines[lines.length - 1] || '';
+            return {
+                type: 'git_diff',
+                description: 'Agent produced code changes',
+                required: true,
+                passed: true,
+                evidence: summaryLine.trim(),
+            };
+        }
+        // Secondary: uncommitted working-tree changes (agent wrote files but did not commit)
+        const unstagedDiff = (0, child_process_1.execSync)(`git diff --stat HEAD`, { cwd: workdir, encoding: 'utf8', timeout }).trim();
+        // Use -uno to exclude untracked files; transcript files and other orchestrator
+        // artifacts dropped into the worktree should not count as "agent made changes".
+        const statusOutput = (0, child_process_1.execSync)(`git status --porcelain -uno`, { cwd: workdir, encoding: 'utf8', timeout }).trim();
+        if (unstagedDiff || statusOutput) {
+            const fileCount = statusOutput ? statusOutput.split('\n').filter(Boolean).length : 0;
+            return {
+                type: 'git_diff',
+                description: 'Agent produced code changes',
+                required: true,
+                passed: true,
+                evidence: `${fileCount} file(s) modified in working tree (uncommitted) — agent completed work without committing`,
+            };
+        }
+        // No committed or uncommitted changes: record for idempotency resolution above
+        return {
+            type: 'git_diff',
+            description: 'Agent produced code changes',
+            required: true,
+            passed: false,
+            reason: `No changes detected since ${baseSha.slice(0, 8)}`,
+        };
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        return {
+            type: 'git_diff',
+            description: 'Agent produced code changes',
+            required: true,
+            passed: false,
+            reason: `git diff failed: ${msg.split('\n')[0]}`,
+        };
+    }
+}
+function checkFileExistence(workdir, expectedFiles) {
+    const missing = [];
+    const present = [];
+    for (const file of expectedFiles) {
+        const fullPath = path.join(workdir, file);
+        if (fs.existsSync(fullPath)) {
+            present.push(file);
+        }
+        else {
+            missing.push(file);
+        }
+    }
+    if (missing.length > 0) {
+        return {
+            type: 'file_existence',
+            description: 'Expected files exist in worktree',
+            required: true,
+            passed: false,
+            reason: `Missing files: ${missing.join(', ')}`,
+            evidence: `${present.length}/${expectedFiles.length} present`,
+        };
+    }
+    return {
+        type: 'file_existence',
+        description: 'Expected files exist in worktree',
+        required: true,
+        passed: true,
+        evidence: `All ${expectedFiles.length} expected file(s) found`,
+    };
+}
+function checkBuildExec(workdir) {
+    const buildCmd = detectBuildCommand(workdir);
+    if (!buildCmd)
+        return null;
+    try {
+        (0, child_process_1.execSync)(buildCmd, {
+            cwd: workdir,
+            encoding: 'utf8',
+            timeout: defaults_1.DEFAULT_COMMAND_TIMEOUT_MS / 2,
+            stdio: ['pipe', 'pipe', 'pipe'],
+        });
+        return {
+            type: 'build_exec',
+            description: `Build succeeded (${buildCmd})`,
+            required: true,
+            passed: true,
+            evidence: `Ran "${buildCmd}" in worktree`,
+        };
+    }
+    catch (err) {
+        const output = extractCommandOutput(err);
+        return {
+            type: 'build_exec',
+            description: `Build failed (${buildCmd})`,
+            required: true,
+            passed: false,
+            reason: last20Lines(output),
+        };
+    }
+}
+function checkTestExec(workdir, workingDir) {
+    const testCmd = detectTestCommand(workdir, workingDir);
+    if (!testCmd)
+        return null;
+    try {
+        (0, child_process_1.execSync)(testCmd, {
+            cwd: workdir,
+            encoding: 'utf8',
+            timeout: defaults_1.DEFAULT_COMMAND_TIMEOUT_MS,
+            stdio: ['pipe', 'pipe', 'pipe'],
+        });
+        return {
+            type: 'test_exec',
+            description: `Tests passed (${testCmd})`,
+            required: true,
+            passed: true,
+            evidence: `Ran "${testCmd}" in worktree`,
+        };
+    }
+    catch (err) {
+        const output = extractCommandOutput(err);
+        if (testCmd === 'npm test' && output.includes('MODULE_NOT_FOUND')) {
+            const fallback = retryTestWithAutoDiscovery(workdir);
+            if (fallback)
+                return fallback;
+        }
+        return {
+            type: 'test_exec',
+            description: `Tests failed (${testCmd})`,
+            required: true,
+            passed: false,
+            reason: last20Lines(output),
+        };
+    }
+}
+function retryTestWithAutoDiscovery(workdir) {
+    const pkgPath = path.join(workdir, 'package.json');
+    try {
+        const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8'));
+        const testScript = pkg.scripts?.test || '';
+        if (!/node\s+--test\s+\S+/.test(testScript))
+            return null;
+    }
+    catch {
+        return null;
+    }
+    const fallbackCmd = 'node --test';
+    try {
+        (0, child_process_1.execSync)(fallbackCmd, {
+            cwd: workdir,
+            encoding: 'utf8',
+            timeout: defaults_1.DEFAULT_COMMAND_TIMEOUT_MS,
+            stdio: ['pipe', 'pipe', 'pipe'],
+        });
+        return {
+            type: 'test_exec',
+            description: `Tests passed (${fallbackCmd}, auto-discovery fallback)`,
+            required: true,
+            passed: true,
+            evidence: `Original npm test hit ESM directory bug; retried with "${fallbackCmd}"`,
+        };
+    }
+    catch {
+        return null;
+    }
+}
+function detectBuildCommand(workdir) {
+    const pkgPath = path.join(workdir, 'package.json');
+    if (fs.existsSync(pkgPath)) {
+        try {
+            const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8'));
+            if (pkg.scripts?.build)
+                return 'npm run build';
+        }
+        catch {
+            // malformed package.json; skip
+        }
+    }
+    const makefilePath = path.join(workdir, 'Makefile');
+    if (fs.existsSync(makefilePath)) {
+        const content = fs.readFileSync(makefilePath, 'utf8');
+        if (/^build\s*:/m.test(content))
+            return 'make build';
+    }
+    return null;
+}
+function resolvePythonBinary(workdir, workingDir) {
+    for (const venvDir of ['venv', '.venv', 'env', '.env']) {
+        const venvPython = path.join(workdir, venvDir, 'bin', 'python');
+        if (fs.existsSync(venvPython))
+            return venvPython;
+    }
+    if (workdir !== workingDir) {
+        for (const venvDir of ['venv', '.venv', 'env', '.env']) {
+            const venvPython = path.join(workingDir, venvDir, 'bin', 'python');
+            if (fs.existsSync(venvPython))
+                return venvPython;
+        }
+    }
+    try {
+        (0, child_process_1.execSync)('python3 --version', { stdio: 'pipe', timeout: defaults_1.DEFAULT_SIGKILL_DELAY_MS });
+        return 'python3';
+    }
+    catch {
+        return 'python';
+    }
+}
+function detectTestCommand(workdir, workingDir) {
+    const pkgPath = path.join(workdir, 'package.json');
+    if (fs.existsSync(pkgPath)) {
+        try {
+            const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8'));
+            const testScript = pkg.scripts?.test || '';
+            const isPlaceholder = /no test(s)? specified/i.test(testScript)
+                || /exit\s+1/.test(testScript) && testScript.includes('echo');
+            if (testScript && !isPlaceholder)
+                return 'npm test';
+        }
+        catch {
+            // malformed package.json; skip
+        }
+    }
+    const makefilePath = path.join(workdir, 'Makefile');
+    // pytest is invoked by the verifier, NOT the agent. When the agent runs
+    // tests inside its worktree, it does so via its own shell commands; those
+    // are transcribed and transcript-parsed by the verifyTests path. This pytest
+    // invocation is the outcome-check path, and it must be isolated from
+    // orchestrator-generated artifact trees:
+    //   - --rootdir="<workdir>" scopes pytest's rootdir search to the repo root
+    //     instead of walking up to a common ancestor with parent worktrees (the
+    //     shape that caused sympy__sympy-12481 to fail on double-collection of
+    //     conftest.py).
+    //   - --ignore paths prevent pytest from descending into orchestrator
+    //     artifact trees whose own conftest.py would re-register options.
+    const pytestCmd = () => {
+        const py = resolvePythonBinary(workdir, workingDir);
+        const runsDir = path.join(workdir, 'runs');
+        const swarmDir = path.join(workdir, '.swarm');
+        return (`${py} -m pytest --rootdir="${workdir}" ` +
+            `--ignore="${runsDir}" --ignore="${swarmDir}"`);
+    };
+    const pyprojectPath = path.join(workdir, 'pyproject.toml');
+    if (fs.existsSync(pyprojectPath)) {
+        const content = fs.readFileSync(pyprojectPath, 'utf8');
+        if (content.includes('[tool.pytest') || content.includes('pytest')) {
+            return pytestCmd();
+        }
+    }
+    const setupCfgPath = path.join(workdir, 'setup.cfg');
+    if (fs.existsSync(setupCfgPath)) {
+        const content = fs.readFileSync(setupCfgPath, 'utf8');
+        if (content.includes('[tool:pytest]')) {
+            return pytestCmd();
+        }
+    }
+    for (const reqFile of ['requirements.txt', 'requirements-dev.txt', 'requirements-test.txt']) {
+        const reqPath = path.join(workdir, reqFile);
+        if (fs.existsSync(reqPath)) {
+            const content = fs.readFileSync(reqPath, 'utf8');
+            if (/^pytest/m.test(content)) {
+                return pytestCmd();
+            }
+        }
+    }
+    if (fs.existsSync(makefilePath)) {
+        const content = fs.readFileSync(makefilePath, 'utf8');
+        if (/^test\s*:/m.test(content))
+            return 'make test';
+    }
+    for (const testDir of ['tests', 'test']) {
+        const testDirPath = path.join(workdir, testDir);
+        if (fs.existsSync(testDirPath) && fs.statSync(testDirPath).isDirectory()) {
+            try {
+                const entries = fs.readdirSync(testDirPath);
+                const hasPyTests = entries.some(f => f.startsWith('test_') && f.endsWith('.py'));
+                if (hasPyTests)
+                    return pytestCmd();
+            }
+            catch {
+                // unreadable dir; skip
+            }
+        }
+    }
+    return null;
+}
+function extractCommandOutput(err) {
+    if (err && typeof err === 'object') {
+        const e = err;
+        const parts = [];
+        if (e.stdout)
+            parts.push(e.stdout);
+        if (e.stderr)
+            parts.push(e.stderr);
+        if (parts.length > 0)
+            return parts.join('\n');
+        if (e.message)
+            return e.message;
+    }
+    return String(err);
+}

package/dist/src/verifier/transcript-checks.js

@@ -0,0 +1,207 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildSummary = buildSummary;
+exports.verifyTests = verifyTests;
+exports.verifyBuild = verifyBuild;
+exports.verifyCommits = verifyCommits;
+exports.verifyAllClaims = verifyAllClaims;
+exports.crossReferenceEvidence = crossReferenceEvidence;
+const hook_generator_1 = require("../hook-generator");
+function buildSummary(checks, passed) {
+    const total = checks.length;
+    const passedCount = checks.filter(c => c.passed).length;
+    const requiredFailed = checks.filter(c => c.required && !c.passed).length;
+    if (passed) {
+        return `${passedCount}/${total} checks passed`;
+    }
+    return `${requiredFailed} required check(s) failed out of ${total} total`;
+}
+function verifyTests(shareIndex) {
+    const testsRun = shareIndex.testsRun;
+    if (testsRun.length === 0) {
+        return {
+            type: 'test',
+            description: 'Tests executed',
+            required: true,
+            passed: false,
+            reason: 'No test commands found in transcript'
+        };
+    }
+    const verifiedTests = testsRun.filter(t => t.verified);
+    if (verifiedTests.length === 0) {
+        return {
+            type: 'test',
+            description: 'Tests executed with output',
+            required: true,
+            passed: false,
+            reason: 'Test commands found but no test output detected',
+            evidence: `Commands: ${testsRun.map(t => t.command).join(', ')}`
+        };
+    }
+    return {
+        type: 'test',
+        description: 'Tests executed successfully',
+        required: true,
+        passed: true,
+        evidence: `${verifiedTests.length} test(s) verified: ${verifiedTests.map(t => t.command).join(', ')}`
+    };
+}
+function verifyBuild(shareIndex) {
+    const builds = shareIndex.buildOperations;
+    if (builds.length === 0) {
+        return {
+            type: 'build',
+            description: 'Build executed',
+            required: true,
+            passed: false,
+            reason: 'No build commands found in transcript'
+        };
+    }
+    const verifiedBuilds = builds.filter(b => b.verified);
+    if (verifiedBuilds.length === 0) {
+        return {
+            type: 'build',
+            description: 'Build succeeded',
+            required: true,
+            passed: false,
+            reason: 'Build commands found but no success output detected',
+            evidence: `Tools: ${builds.map(b => b.tool).join(', ')}`
+        };
+    }
+    return {
+        type: 'build',
+        description: 'Build succeeded',
+        required: true,
+        passed: true,
+        evidence: `Verified builds: ${verifiedBuilds.map(b => b.tool).join(', ')}`
+    };
+}
+function verifyCommits(shareIndex) {
+    const commits = shareIndex.gitCommits;
+    if (commits.length === 0) {
+        return {
+            type: 'commit',
+            description: 'Git commits made',
+            required: true,
+            passed: false,
+            reason: 'No git commits found in transcript'
+        };
+    }
+    const verifiedCommits = commits.filter(c => c.verified);
+    if (verifiedCommits.length === 0) {
+        return {
+            type: 'commit',
+            description: 'Git commits verified',
+            required: true,
+            passed: false,
+            reason: 'Commit messages found but not verified in output'
+        };
+    }
+    return {
+        type: 'commit',
+        description: 'Git commits verified',
+        required: true,
+        passed: true,
+        evidence: `${verifiedCommits.length} commit(s): ${verifiedCommits.map(c => c.message).slice(0, 3).join('; ')}${verifiedCommits.length > 3 ? '...' : ''}`
+    };
+}
+function verifyAllClaims(shareIndex) {
+    const checks = [];
+    const unverifiedClaims = [];
+    shareIndex.claims.forEach(claim => {
+        if (!claim.verified) {
+            checks.push({
+                type: 'claim',
+                description: `Verify claim: "${claim.claim.substring(0, 50)}..."`,
+                required: false,
+                passed: false,
+                reason: claim.evidence || 'No evidence found',
+                evidence: claim.claim
+            });
+            unverifiedClaims.push(claim.claim);
+        }
+    });
+    return { checks, unverifiedClaims };
+}
+function crossReferenceEvidence(shareIndex, evidenceLogPath) {
+    const fs = require('fs');
+    // Hook evidence is only produced by adapters that load hooks from
+    // <gitRoot>/.github/hooks/ (currently only Copilot CLI). If the file does
+    // not exist, the adapter is not wired for hooks; suppress the check
+    // instead of synthesizing a failed "should have but didn't" record that
+    // misleads users on claude-code, codex, or claude-code-teams runs.
+    if (!fs.existsSync(evidenceLogPath)) {
+        return [];
+    }
+    const hookGen = new hook_generator_1.HookGenerator();
+    const entries = hookGen.parseEvidenceLog(evidenceLogPath);
+    if (entries.length === 0) {
+        return [{
+                type: 'claim',
+                description: 'Hook evidence log exists and is non-empty',
+                required: false,
+                passed: false,
+                reason: `Hook evidence log at ${evidenceLogPath} is empty; hooks ran but recorded no events.`
+            }];
+    }
+    const checks = [];
+    const errorEntries = entries.filter(e => e.event === 'errorOccurred');
+    const transcriptHasErrors = shareIndex.testsRun.some(t => !t.verified)
+        || shareIndex.buildOperations.some(b => !b.verified)
+        || shareIndex.claims.some(c => !c.verified);
+    if (errorEntries.length > 0 && !transcriptHasErrors) {
+        checks.push({
+            type: 'claim',
+            description: 'Hook error evidence cross-references transcript',
+            required: true,
+            passed: false,
+            evidence: `Hooks captured ${errorEntries.length} error(s) but transcript reports none`,
+            reason: `Hook evidence contradicts transcript: ${errorEntries.length} error(s) captured by hooks were not mentioned in transcript`
+        });
+    }
+    else if (errorEntries.length > 0) {
+        checks.push({
+            type: 'claim',
+            description: 'Hook error evidence cross-references transcript',
+            required: false,
+            passed: true,
+            evidence: `Both hooks (${errorEntries.length}) and transcript acknowledge errors`
+        });
+    }
+    const hookFiles = new Set(entries
+        .filter(e => e.filePath)
+        .map(e => e.filePath));
+    const transcriptFiles = new Set(shareIndex.changedFiles || []);
+    const scopeViolations = entries.filter(e => e.event === 'scope_violation');
+    if (scopeViolations.length > 0) {
+        const violatedFiles = scopeViolations.map(v => v.filePath || 'unknown').join(', ');
+        const agents = [...new Set(scopeViolations.map(v => v.agentName || 'unknown'))];
+        const rules = [...new Set(scopeViolations.map(v => v.boundaryRule || 'unknown'))];
+        checks.push({
+            type: 'claim',
+            description: 'No scope violations detected during execution',
+            required: true,
+            passed: false,
+            evidence: `${scopeViolations.length} scope violation(s) logged by hooks`,
+            reason: `Agent ${agents.join(', ')} wrote to files outside its declared scope: ${violatedFiles}. Violated boundary rule(s): ${rules.join(', ')}`
+        });
+    }
+    const hookOnlyFiles = [...hookFiles].filter(f => !transcriptFiles.has(f));
+    if (hookOnlyFiles.length > 0) {
+        checks.push({
+            type: 'claim',
+            description: 'Hook file evidence has unmentioned files',
+            required: false,
+            passed: true,
+            evidence: `Hooks captured ${hookOnlyFiles.length} file(s) not mentioned in transcript: ${hookOnlyFiles.slice(0, 5).join(', ')}`
+        });
+    }
+    checks.push({
+        type: 'claim',
+        description: 'Hook evidence corroborates transcript',
+        required: false,
+        passed: true,
+        evidence: `${entries.length} hook evidence entries, ${hookFiles.size} files tracked`
+    });
+    return checks;
+}