swarm-orchestrator 7.0.0-alpha.0

package/dist/src/verifier/verification-reporters.js

@@ -0,0 +1,148 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateVerificationReport = generateVerificationReport;
+exports.commitVerificationReport = commitVerificationReport;
+const child_process_1 = require("child_process");
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+async function runGitCommand(workingDir, args) {
+    return new Promise((resolve, reject) => {
+        const proc = (0, child_process_1.spawn)('git', args, { cwd: workingDir });
+        let stdout = '';
+        let stderr = '';
+        proc.stdout?.on('data', (data) => {
+            stdout += data.toString();
+        });
+        proc.stderr?.on('data', (data) => {
+            stderr += data.toString();
+        });
+        proc.on('close', (code) => {
+            if (code === 0)
+                resolve(stdout);
+            else
+                reject(new Error(stderr || `Git command failed with code ${code}`));
+        });
+        proc.on('error', (err) => reject(err));
+    });
+}
+/**
+ * Render a VerificationResult as a Markdown report and persist it.
+ */
+async function generateVerificationReport(result, outputPath) {
+    const lines = [];
+    lines.push('# Verification Report');
+    lines.push('');
+    lines.push(`**Step**: ${result.stepNumber}`);
+    lines.push(`**Agent**: ${result.agentName}`);
+    lines.push(`**Status**: ${result.passed ? '✅ PASSED' : '❌ FAILED'}`);
+    lines.push(`**Timestamp**: ${result.timestamp}`);
+    lines.push(`**Transcript**: ${result.transcriptPath}`);
+    lines.push('');
+    lines.push('## Verification Checks');
+    lines.push('');
+    result.checks.forEach((check) => {
+        const icon = check.passed ? '✅' : '❌';
+        const req = check.required ? '(required)' : '(optional)';
+        lines.push(`### ${icon} ${check.description} ${req}`);
+        lines.push('');
+        lines.push(`**Type**: ${check.type}`);
+        lines.push(`**Passed**: ${check.passed}`);
+        if (check.evidence)
+            lines.push(`**Evidence**: ${check.evidence}`);
+        if (check.reason)
+            lines.push(`**Reason**: ${check.reason}`);
+        lines.push('');
+    });
+    if (result.unverifiedClaims.length > 0) {
+        lines.push('## ⚠️ Unverified Claims (Drift Detection)');
+        lines.push('');
+        lines.push('The following claims were made without supporting evidence:');
+        lines.push('');
+        result.unverifiedClaims.forEach((claim) => {
+            lines.push(`- ${claim}`);
+        });
+        lines.push('');
+    }
+    lines.push('## Summary');
+    lines.push('');
+    const passedCount = result.checks.filter((c) => c.passed).length;
+    const totalCount = result.checks.length;
+    lines.push(`**Checks Passed**: ${passedCount}/${totalCount}`);
+    lines.push(`**Unverified Claims**: ${result.unverifiedClaims.length}`);
+    lines.push('');
+    if (!result.passed) {
+        lines.push('**Action Required**: This step failed verification. Review the issues above and retry.');
+    }
+    else {
+        lines.push('**Result**: All required checks passed. Step verified successfully.');
+    }
+    const reportContent = lines.join('\n');
+    const dir = path.dirname(outputPath);
+    if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir, { recursive: true });
+    }
+    fs.writeFileSync(outputPath, reportContent, 'utf8');
+}
+/**
+ * Commit the verification report with a natural-language message.
+ * Skips silently when the report lives outside the target worktree
+ * (e.g. bootstrap mode writing into the orchestrator's own runs/).
+ */
+async function commitVerificationReport(reportPath, workingDir, stepNumber, agentName, passed, logger) {
+    const status = passed ? 'verified' : 'failed verification';
+    const messages = [
+        `verify step ${stepNumber} (${agentName}) - ${status}`,
+        `add verification report for step ${stepNumber}`,
+        `verification: step ${stepNumber} ${status}`,
+        `step ${stepNumber} verification ${passed ? 'passed' : 'failed'}`,
+    ];
+    const message = messages[Math.floor(Math.random() * messages.length)];
+    const resolvedReport = path.resolve(reportPath);
+    const resolvedWorkDir = path.resolve(workingDir);
+    if (!resolvedReport.startsWith(resolvedWorkDir + path.sep)) {
+        return;
+    }
+    try {
+        await runGitCommand(workingDir, ['add', '-f', reportPath]);
+        await runGitCommand(workingDir, ['commit', '-m', message]);
+    }
+    catch (err) {
+        const error = err;
+        if (!error.message.includes('nothing to commit')) {
+            logger.warn(`  ⚠️  Could not commit verification report: ${error.message.split('\n')[0]}`);
+        }
+    }
+}

package/dist/src/verifier-engine.js

@@ -0,0 +1,491 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VerifierEngine = void 0;
+exports.last20Lines = last20Lines;
+exports.buildFailureContext = buildFailureContext;
+const child_process_1 = require("child_process");
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const share_parser_1 = __importDefault(require("./share-parser"));
+const defaults_1 = require("./defaults");
+const outcome_checks_1 = require("./verifier/outcome-checks");
+const transcript_checks_1 = require("./verifier/transcript-checks");
+const verification_reporters_1 = require("./verifier/verification-reporters");
+const logger_1 = require("./logger");
+const logger = (0, logger_1.getLogger)('verifier-engine');
+/**
+ * Truncate output to the last 20 lines for error display.
+ * Returns the full string if it has 20 lines or fewer.
+ */
+function last20Lines(output) {
+    const lines = output.split('\n');
+    if (lines.length <= 20)
+        return output;
+    return '...\n' + lines.slice(-20).join('\n');
+}
+// Ordering for failure context: more actionable types first
+const FAILURE_TYPE_PRIORITY = {
+    file_existence: 0,
+    test_exec: 1,
+    build_exec: 2,
+    git_diff: 3,
+};
+/**
+ * Build a structured failure context string from failed checks.
+ * Ordered by actionability and truncated to ~500 tokens (2000 chars).
+ */
+function buildFailureContext(checks) {
+    const failed = checks.filter(c => !c.passed);
+    if (failed.length === 0)
+        return '';
+    failed.sort((a, b) => {
+        const pa = FAILURE_TYPE_PRIORITY[a.type] ?? 10;
+        const pb = FAILURE_TYPE_PRIORITY[b.type] ?? 10;
+        return pa - pb;
+    });
+    const lines = failed.map(c => `- ${c.type}: ${c.reason || c.description}`);
+    const joined = lines.join('\n');
+    // ~500 tokens at 4 chars/token = 2000 chars
+    if (joined.length > defaults_1.DEFAULT_FAILURE_CONTEXT_CHARS) {
+        return joined.slice(0, defaults_1.DEFAULT_FAILURE_CONTEXT_CHARS) + '\n... (truncated)';
+    }
+    return joined;
+}
+/**
+ * Verifier Engine - proactive real-time verification and drift prevention
+ * Validates agent claims against actual evidence in transcripts
+ */
+class VerifierEngine {
+    shareParser;
+    workingDir;
+    constructor(workingDir) {
+        this.workingDir = workingDir || process.cwd();
+        this.shareParser = new share_parser_1.default();
+    }
+    /**
+     * Verify a completed step by analyzing its transcript
+     */
+    async verifyStep(stepNumber, agentName, transcriptPath, requirements, preParsedIndex, evidenceLogPath, outcomeOpts) {
+        const checks = [];
+        const unverifiedClaims = [];
+        // Read and parse transcript
+        if (!fs.existsSync(transcriptPath)) {
+            return {
+                stepNumber,
+                agentName,
+                passed: false,
+                checks: [{
+                        type: 'claim',
+                        description: 'Transcript file exists',
+                        required: true,
+                        passed: false,
+                        reason: `Transcript not found: ${transcriptPath}`
+                    }],
+                unverifiedClaims: [],
+                timestamp: new Date().toISOString(),
+                transcriptPath
+            };
+        }
+        // Skip re-parse when the caller already parsed the transcript
+        const shareIndex = preParsedIndex ?? (() => {
+            const transcriptContent = fs.readFileSync(transcriptPath, 'utf8');
+            return this.shareParser.parse(transcriptContent);
+        })();
+        // Verify tests if required
+        if (requirements?.requireTests) {
+            const testCheck = this.verifyTests(shareIndex);
+            checks.push(testCheck);
+        }
+        // Verify build if required
+        if (requirements?.requireBuild) {
+            const buildCheck = this.verifyBuild(shareIndex);
+            checks.push(buildCheck);
+        }
+        // Verify commits if required
+        if (requirements?.requireCommits) {
+            const commitCheck = this.verifyCommits(shareIndex);
+            checks.push(commitCheck);
+        }
+        // Check for unverified claims (drift detection) - these are warnings only
+        const claimCheck = this.verifyAllClaims(shareIndex);
+        checks.push(...claimCheck.checks);
+        unverifiedClaims.push(...claimCheck.unverifiedClaims);
+        // Cross-reference hook evidence against transcript claims (defense in depth)
+        if (evidenceLogPath) {
+            const hookEvidence = this.crossReferenceEvidence(shareIndex, evidenceLogPath);
+            checks.push(...hookEvidence);
+        }
+        // Outcome-based checks: run when the caller provides worktree info
+        if (outcomeOpts) {
+            const outcomeChecks = this.runOutcomeChecks(outcomeOpts);
+            checks.push(...outcomeChecks);
+            // Demote transcript-parsed test/build checks when real execution checks exist
+            const hasOutcomeExecChecks = checks.some(c => c.type === 'build_exec' || c.type === 'test_exec');
+            if (hasOutcomeExecChecks) {
+                for (const c of checks) {
+                    if (c.type === 'build' || c.type === 'test') {
+                        c.required = false;
+                    }
+                }
+            }
+        }
+        // Overall pass/fail: every required check must pass (vacuously true when none are required)
+        const passed = checks.every(c => !c.required || c.passed);
+        // Build failure context for repair agent consumption
+        const failureContext = passed ? undefined : buildFailureContext(checks);
+        const summary = this.buildSummary(checks, passed);
+        // Apply graceful degradation if enabled and verification failed
+        let gracefulFailure = false;
+        let degradationReason;
+        if (!passed && requirements?.gracefulDegradation) {
+            gracefulFailure = true;
+            const failedRequired = checks.filter(c => c.required && !c.passed);
+            degradationReason = `Verification failed but graceful degradation enabled. Failed checks: ${failedRequired.map(c => c.type).join(', ')}`;
+        }
+        const result = {
+            stepNumber,
+            agentName,
+            passed: passed || gracefulFailure,
+            checks,
+            unverifiedClaims,
+            timestamp: new Date().toISOString(),
+            transcriptPath,
+            gracefulFailure,
+            degradationReason,
+            failureContext,
+            summary,
+            baseSha: outcomeOpts?.baseSha,
+        };
+        return result;
+    }
+    runOutcomeChecks(opts) {
+        return (0, outcome_checks_1.runOutcomeChecks)(opts, this.workingDir);
+    }
+    buildSummary(checks, passed) {
+        return (0, transcript_checks_1.buildSummary)(checks, passed);
+    }
+    verifyTests(shareIndex) {
+        return (0, transcript_checks_1.verifyTests)(shareIndex);
+    }
+    verifyBuild(shareIndex) {
+        return (0, transcript_checks_1.verifyBuild)(shareIndex);
+    }
+    verifyCommits(shareIndex) {
+        return (0, transcript_checks_1.verifyCommits)(shareIndex);
+    }
+    verifyAllClaims(shareIndex) {
+        return (0, transcript_checks_1.verifyAllClaims)(shareIndex);
+    }
+    /**
+     * Verify expected outputs by checking files and content in the worktree.
+     * Each expectedOutput string is parsed for file paths and key terms,
+     * then verified against the actual file system.
+     */
+    verifyExpectedOutputs(stepNumber, agentName, expectedOutputs, worktreeDir) {
+        const checks = [];
+        // Regex to extract file paths from expected output descriptions
+        const filePathRegex = /\b((?:src|test|tests|server|config|public)\/[\w./-]+\.\w+|[\w-]+\.\w{2,4})\b/g;
+        // Key terms that can be grepped in source files
+        const termPatterns = [
+            { term: 'aria-label', regex: /aria-label/i },
+            { term: 'aria-live', regex: /aria-live/i },
+            { term: 'role="alert"', regex: /role\s*=\s*["']alert["']/i },
+            { term: 'focus-visible', regex: /:focus-visible/i },
+            { term: 'skip-link', regex: /skip-link|skip.to.content|#main-content/i },
+            { term: 'ErrorBoundary', regex: /ErrorBoundary/i },
+            { term: 'AbortController', regex: /AbortController/i },
+            { term: 'retry', regex: /retry|backoff/i },
+            { term: 'responsive breakpoint', regex: /@media\s*\(/i },
+            { term: 'heading hierarchy', regex: /<h[12]/i },
+            { term: 'skeleton shimmer', regex: /skeleton|shimmer/i },
+            { term: 'animation', regex: /animation\s*[:={]/i }
+        ];
+        for (const expected of expectedOutputs) {
+            const filePaths = [];
+            let match;
+            const pathRegex = new RegExp(filePathRegex.source, 'g');
+            while ((match = pathRegex.exec(expected)) !== null) {
+                // Skip common false positives like version numbers
+                if (/^\d+\.\d+/.test(match[1]))
+                    continue;
+                filePaths.push(match[1]);
+            }
+            if (filePaths.length > 0) {
+                // File-existence check: verify at least one mentioned file exists
+                const existingFiles = filePaths.filter(fp => {
+                    const fullPath = path.join(worktreeDir, fp);
+                    return fs.existsSync(fullPath);
+                });
+                if (existingFiles.length > 0) {
+                    checks.push({
+                        type: 'output',
+                        description: `Expected files present: ${existingFiles.join(', ')}`,
+                        required: false,
+                        passed: true,
+                        evidence: `${existingFiles.length}/${filePaths.length} expected files found in worktree`
+                    });
+                }
+                else {
+                    checks.push({
+                        type: 'output',
+                        description: `Expected files: ${filePaths.join(', ')}`,
+                        required: false,
+                        passed: false,
+                        reason: `None of the expected files found in ${worktreeDir}`,
+                        evidence: `Looked for: ${filePaths.join(', ')}`
+                    });
+                }
+                // Content check: grep existing files for key terms mentioned in expected output
+                const lowerExpected = expected.toLowerCase();
+                for (const { term, regex } of termPatterns) {
+                    if (!lowerExpected.includes(term.toLowerCase().split(/[="]/)[0]))
+                        continue;
+                    const found = existingFiles.some(fp => {
+                        try {
+                            const content = fs.readFileSync(path.join(worktreeDir, fp), 'utf8');
+                            return regex.test(content);
+                        }
+                        catch {
+                            // File may be binary or permission-restricted; skip it
+                            return false;
+                        }
+                    });
+                    if (found) {
+                        checks.push({
+                            type: 'output',
+                            description: `"${term}" found in output files`,
+                            required: false,
+                            passed: true,
+                            evidence: `Matched ${term} pattern in generated code`
+                        });
+                    }
+                }
+            }
+            else {
+                // No file paths extracted; try to match key terms across all changed files
+                const lowerExpected = expected.toLowerCase();
+                const matchedTerms = [];
+                for (const { term, regex } of termPatterns) {
+                    if (!lowerExpected.includes(term.toLowerCase().split(/[="]/)[0]))
+                        continue;
+                    // Search all non-binary files in worktree src/ and test/
+                    const searchDirs = ['src', 'test', 'tests', 'server'].map(d => path.join(worktreeDir, d));
+                    let found = false;
+                    for (const dir of searchDirs) {
+                        if (!fs.existsSync(dir))
+                            continue;
+                        try {
+                            const files = fs.readdirSync(dir, { recursive: true, encoding: 'utf8' });
+                            for (const file of files) {
+                                const fullPath = path.join(dir, file);
+                                try {
+                                    const stat = fs.statSync(fullPath);
+                                    if (!stat.isFile() || stat.size > 256 * 1024)
+                                        continue;
+                                    const content = fs.readFileSync(fullPath, 'utf8');
+                                    if (regex.test(content)) {
+                                        found = true;
+                                        break;
+                                    }
+                                }
+                                catch {
+                                    // Binary or permission-restricted file; skip
+                                }
+                            }
+                        }
+                        catch {
+                            // Directory unreadable (permissions, broken symlink); skip
+                        }
+                        if (found)
+                            break;
+                    }
+                    if (found)
+                        matchedTerms.push(term);
+                }
+                if (matchedTerms.length > 0) {
+                    checks.push({
+                        type: 'output',
+                        description: `Expected features verified: ${matchedTerms.join(', ')}`,
+                        required: false,
+                        passed: true,
+                        evidence: `Found evidence for: ${matchedTerms.join(', ')}`
+                    });
+                }
+            }
+        }
+        return checks;
+    }
+    /**
+     * Generate verification report and save to file
+     */
+    async generateVerificationReport(result, outputPath) {
+        return (0, verification_reporters_1.generateVerificationReport)(result, outputPath);
+    }
+    /**
+     * Rollback changes from a failed step
+     */
+    async rollback(stepNumber, branchName, filesChanged, baseBranch) {
+        const filesRestored = [];
+        try {
+            // If on a branch, switch back and delete the branch. Use the caller's
+            // explicit base branch when provided; otherwise resolve it from origin/HEAD
+            // so this works on `master` repos too, not just `main`. Never resolve to
+            // the branch we're about to delete — git refuses to delete the current
+            // branch, so the fallback must exclude it.
+            if (branchName) {
+                const target = baseBranch ?? this.resolveBaseBranch(branchName);
+                await this.runGitCommand(['checkout', target]);
+                await this.runGitCommand(['branch', '-D', branchName]);
+            }
+            // Reset any uncommitted changes
+            await this.runGitCommand(['reset', '--hard', 'HEAD']);
+            // If specific files were changed, restore them
+            if (filesChanged && filesChanged.length > 0) {
+                for (const file of filesChanged) {
+                    try {
+                        await this.runGitCommand(['checkout', 'HEAD', '--', file]);
+                        filesRestored.push(file);
+                    }
+                    catch {
+                        // File does not exist in HEAD (new file from the agent); nothing to restore
+                    }
+                }
+            }
+            return {
+                success: true,
+                filesRestored
+            };
+        }
+        catch (error) {
+            const err = error;
+            return {
+                success: false,
+                filesRestored,
+                error: err.message
+            };
+        }
+    }
+    /**
+     * Resolve the repo's integration branch synchronously. Mirrors
+     * WorktreeManager.resolveDefaultBranch's contract: origin/HEAD first, then
+     * the currently checked-out branch, then the literal "main" as a last resort.
+     *
+     * @param excludeBranch When rollback() is deleting a branch, the caller
+     *        passes that branch here so the current-branch fallback never
+     *        resolves to it (git refuses to delete the checked-out branch).
+     */
+    resolveBaseBranch(excludeBranch) {
+        try {
+            const head = (0, child_process_1.execSync)('git symbolic-ref refs/remotes/origin/HEAD', {
+                cwd: this.workingDir,
+                encoding: 'utf8',
+                stdio: ['pipe', 'pipe', 'pipe'],
+            }).trim();
+            const prefix = 'refs/remotes/origin/';
+            if (head.startsWith(prefix))
+                return head.slice(prefix.length);
+        }
+        catch {
+            // origin/HEAD not set; fall through
+        }
+        try {
+            const current = (0, child_process_1.execSync)('git branch --show-current', {
+                cwd: this.workingDir,
+                encoding: 'utf8',
+                stdio: ['pipe', 'pipe', 'pipe'],
+            }).trim();
+            if (current && current !== excludeBranch)
+                return current;
+        }
+        catch {
+            // fall through
+        }
+        return 'main';
+    }
+    /**
+     * Run a git command
+     */
+    async runGitCommand(args) {
+        return new Promise((resolve, reject) => {
+            const proc = (0, child_process_1.spawn)('git', args, {
+                cwd: this.workingDir
+            });
+            let stdout = '';
+            let stderr = '';
+            if (proc.stdout) {
+                proc.stdout.on('data', (data) => {
+                    stdout += data.toString();
+                });
+            }
+            if (proc.stderr) {
+                proc.stderr.on('data', (data) => {
+                    stderr += data.toString();
+                });
+            }
+            proc.on('close', (code) => {
+                if (code === 0) {
+                    resolve(stdout);
+                }
+                else {
+                    reject(new Error(stderr || `Git command failed with code ${code}`));
+                }
+            });
+            proc.on('error', (err) => {
+                reject(err);
+            });
+        });
+    }
+    /**
+     * Commit verification report with natural message
+     * Skips if file is in a gitignored path (e.g., runs/)
+     */
+    async commitVerificationReport(reportPath, stepNumber, agentName, passed) {
+        return (0, verification_reporters_1.commitVerificationReport)(reportPath, this.workingDir, stepNumber, agentName, passed, logger);
+    }
+    /**
+     * Cross-reference hook-captured evidence with transcript-based claims.
+     * If hooks recorded errors or scope violations that the transcript does not
+     * mention, the step fails (defense in depth).
+     */
+    crossReferenceEvidence(shareIndex, evidenceLogPath) {
+        return (0, transcript_checks_1.crossReferenceEvidence)(shareIndex, evidenceLogPath);
+    }
+}
+exports.VerifierEngine = VerifierEngine;
+exports.default = VerifierEngine;