vibecodingmachine-core 2026.2.20-438 → 2026.2.26-1739

package/src/agents/verification/VerificationRunner.js

@@ -0,0 +1,433 @@
+/**
+ * Verification Test Runner
+ * 
+ * Orchestrates multiple verification tests.
+ * Follows constitutional requirements: <555 lines, test-first approach.
+ */
+
+const BaseVerifier = require('./BaseVerifier');
+const FileOperationVerifier = require('./FileOperationVerifier');
+const CommandVerifier = require('./CommandVerifier');
+const { VerificationType } = require('../VerificationType');
+
+/**
+ * Verification test runner class
+ */
+class VerificationRunner {
+  /**
+   * Create verification runner instance
+   * @param {Object} config - Runner configuration
+   */
+  constructor(config = {}) {
+    this.logger = config.logger || null;
+    this.parallel = config.parallel !== false; // Default to parallel
+    this.maxConcurrency = config.maxConcurrency || 3;
+    this.timeout = config.timeout || 300000; // 5 minutes default
+    this.continueOnFailure = config.continueOnFailure !== false; // Default to continue
+  }
+
+  /**
+   * Run verification tests for an agent
+   * @param {Object} agent - Agent configuration
+   * @param {Array<Object>} testConfigs - Test configurations
+   * @returns {Promise<Object>} - Verification results
+   */
+  async runVerificationTests(agent, testConfigs) {
+    if (this.logger) {
+      await this.logger.info('Starting verification tests', {
+        agentId: agent.id,
+        testCount: testConfigs.length,
+        parallel: this.parallel
+      });
+    }
+
+    const startTime = Date.now();
+    const results = {
+      agentId: agent.id,
+      totalTests: testConfigs.length,
+      passed: 0,
+      failed: 0,
+      skipped: 0,
+      duration: 0,
+      tests: [],
+      summary: {
+        allPassed: false,
+        hasFailures: false,
+        hasSkipped: false
+      }
+    };
+
+    try {
+      if (this.parallel) {
+        // Run tests in parallel with concurrency limit
+        const batches = this.createBatches(testConfigs, this.maxConcurrency);
+        
+        for (const batch of batches) {
+          const batchResults = await this.runBatchInParallel(batch, agent);
+          this.mergeResults(results, batchResults);
+        }
+      } else {
+        // Run tests sequentially
+        for (const testConfig of testConfigs) {
+          const result = await this.runSingleTest(testConfig, agent);
+          this.addResult(results, result);
+          
+          // Stop if test failed and continueOnFailure is false
+          if (!result.passed && !this.continueOnFailure) {
+            break;
+          }
+        }
+      }
+    } catch (error) {
+      if (this.logger) {
+        await this.logger.error('Verification runner error', {
+          agentId: agent.id,
+          error: error.message
+        });
+      }
+      
+      // Add error as a failed test
+      const errorResult = {
+        id: this.generateId(),
+        agentId: agent.id,
+        type: 'RUNNER_ERROR',
+        description: 'Verification runner error',
+        passed: false,
+        error: error.message,
+        duration: Date.now() - startTime,
+        timestamp: new Date().toISOString()
+      };
+      
+      this.addResult(results, errorResult);
+    }
+
+    results.duration = Date.now() - startTime;
+    this.calculateSummary(results);
+
+    if (this.logger) {
+      await this.logger.info('Verification tests completed', {
+        agentId: agent.id,
+        totalTests: results.totalTests,
+        passed: results.passed,
+        failed: results.failed,
+        skipped: results.skipped,
+        duration: results.duration,
+        allPassed: results.summary.allPassed
+      });
+    }
+
+    return results;
+  }
+
+  /**
+   * Run a batch of tests in parallel
+   * @param {Array<Object>} batch - Batch of test configurations
+   * @param {Object} agent - Agent configuration
+   * @returns {Promise<Array<Object>>} - Batch results
+   */
+  async runBatchInParallel(batch, agent) {
+    const promises = batch.map(testConfig => 
+      this.runSingleTest(testConfig, agent)
+    );
+
+    return Promise.allSettled(promises);
+  }
+
+  /**
+   * Run a single verification test
+   * @param {Object} testConfig - Test configuration
+   * @param {Object} agent - Agent configuration
+   * @returns {Promise<Object>} - Test result
+   */
+  async runSingleTest(testConfig, agent) {
+    try {
+      const verifier = this.createVerifier(testConfig);
+      const context = { agentId: agent.id, agentCommand: agent.verificationCommands?.[0] };
+      
+      const result = await verifier.verify({ context });
+      
+      return {
+        ...result,
+        testConfig,
+        verifierType: testConfig.type
+      };
+    } catch (error) {
+      return {
+        id: this.generateId(),
+        agentId: agent.id,
+        type: testConfig.type || 'UNKNOWN',
+        description: testConfig.description || 'Unknown test',
+        passed: false,
+        error: error.message,
+        duration: 0,
+        timestamp: new Date().toISOString(),
+        testConfig,
+        verifierType: testConfig.type
+      };
+    }
+  }
+
+  /**
+   * Create verifier instance based on configuration
+   * @param {Object} testConfig - Test configuration
+   * @returns {BaseVerifier} - Verifier instance
+   */
+  createVerifier(testConfig) {
+    switch (testConfig.type) {
+      case VerificationType.FILE_OPERATION:
+        return FileOperationVerifier.fromConfig(testConfig, this.logger);
+      
+      case VerificationType.COMMAND_EXECUTION:
+        return CommandVerifier.fromConfig(testConfig, this.logger);
+      
+      case VerificationType.API_CALL:
+        // TODO: Implement ApiCallVerifier
+        throw new Error('ApiCallVerifier not yet implemented');
+      
+      case VerificationType.NETWORK_TEST:
+        // TODO: Implement NetworkVerifier
+        throw new Error('NetworkVerifier not yet implemented');
+      
+      default:
+        throw new Error(`Unknown verification type: ${testConfig.type}`);
+    }
+  }
+
+  /**
+   * Create batches for parallel execution
+   * @param {Array<Object>} items - Items to batch
+   * @param {number} batchSize - Maximum batch size
+   * @returns {Array<Array<Object>>} - Array of batches
+   */
+  createBatches(items, batchSize) {
+    const batches = [];
+    
+    for (let i = 0; i < items.length; i += batchSize) {
+      batches.push(items.slice(i, i + batchSize));
+    }
+    
+    return batches;
+  }
+
+  /**
+   * Add result to results collection
+   * @param {Object} results - Results collection
+   * @param {Object} result - New result to add
+   */
+  addResult(results, result) {
+    results.tests.push(result);
+    
+    if (result.passed) {
+      results.passed++;
+    } else {
+      results.failed++;
+    }
+  }
+
+  /**
+   * Merge batch results into main results
+   * @param {Object} results - Main results
+   * @param {Array<Object>} batchResults - Batch results
+   */
+  mergeResults(results, batchResults) {
+    for (const batchResult of batchResults) {
+      if (batchResult.status === 'fulfilled') {
+        this.addResult(results, batchResult.value);
+      } else {
+        // Handle rejected promise
+        const errorResult = {
+          id: this.generateId(),
+          agentId: results.agentId,
+          type: 'PROMISE_REJECTED',
+          description: 'Test execution promise rejected',
+          passed: false,
+          error: batchResult.reason?.message || 'Unknown error',
+          duration: 0,
+          timestamp: new Date().toISOString()
+        };
+        
+        this.addResult(results, errorResult);
+      }
+    }
+  }
+
+  /**
+   * Calculate summary statistics
+   * @param {Object} results - Results collection
+   */
+  calculateSummary(results) {
+    results.summary = {
+      allPassed: results.failed === 0 && results.passed === results.totalTests,
+      hasFailures: results.failed > 0,
+      hasSkipped: results.skipped > 0,
+      passRate: results.totalTests > 0 ? (results.passed / results.totalTests) * 100 : 0,
+      failureRate: results.totalTests > 0 ? (results.failed / results.totalTests) * 100 : 0
+    };
+  }
+
+  /**
+   * Generate unique test ID
+   * @returns {string} - Unique identifier
+   */
+  generateId() {
+    const timestamp = Date.now();
+    const random = Math.random().toString(36).substr(2, 9);
+    return `verify-${timestamp}-${random}`;
+  }
+
+  /**
+   * Run verification tests with timeout protection
+   * @param {Object} agent - Agent configuration
+   * @param {Array<Object>} testConfigs - Test configurations
+   * @param {number} timeout - Overall timeout in milliseconds
+   * @returns {Promise<Object>} - Verification results with timeout handling
+   */
+  async runVerificationTestsWithTimeout(agent, testConfigs, timeout = this.timeout) {
+    return new Promise((resolve, reject) => {
+      const timeoutId = setTimeout(() => {
+        reject(new Error(`Verification tests timed out after ${timeout}ms`));
+      }, timeout);
+
+      this.runVerificationTests(agent, testConfigs)
+        .then(results => {
+          clearTimeout(timeoutId);
+          resolve(results);
+        })
+        .catch(error => {
+          clearTimeout(timeoutId);
+          reject(error);
+        });
+    });
+  }
+
+  /**
+   * Get test execution statistics
+   * @param {Array<Object>} results - Test results
+   * @returns {Object} - Statistics
+   */
+  getTestStatistics(results) {
+    const stats = {
+      totalTests: results.length,
+      passedTests: results.filter(r => r.passed).length,
+      failedTests: results.filter(r => !r.passed).length,
+      averageDuration: 0,
+      minDuration: Infinity,
+      maxDuration: 0,
+      testTypes: {},
+      errorTypes: {}
+    };
+
+    // Calculate duration statistics
+    const durations = results
+      .filter(r => r.duration > 0)
+      .map(r => r.duration);
+    
+    if (durations.length > 0) {
+      stats.averageDuration = durations.reduce((a, b) => a + b, 0) / durations.length;
+      stats.minDuration = Math.min(...durations);
+      stats.maxDuration = Math.max(...durations);
+    }
+
+    // Count test types
+    for (const result of results) {
+      stats.testTypes[result.type] = (stats.testTypes[result.type] || 0) + 1;
+      
+      if (result.error) {
+        const errorType = result.error.split(':')[0] || 'UNKNOWN';
+        stats.errorTypes[errorType] = (stats.errorTypes[errorType] || 0) + 1;
+      }
+    }
+
+    return stats;
+  }
+
+  /**
+   * Filter tests by type
+   * @param {Array<Object>} tests - Test results
+   * @param {string} type - Test type to filter by
+   * @returns {Array<Object>} - Filtered tests
+   */
+  filterTestsByType(tests, type) {
+    return tests.filter(test => test.type === type);
+  }
+
+  /**
+   * Get failed tests
+   * @param {Array<Object>} tests - Test results
+   * @returns {Array<Object>} - Failed tests
+   */
+  getFailedTests(tests) {
+    return tests.filter(test => !test.passed);
+  }
+
+  /**
+   * Get passed tests
+   * @param {Array<Object>} tests - Test results
+   * @returns {Array<Object>} - Passed tests
+   */
+  getPassedTests(tests) {
+    return tests.filter(test => test.passed);
+  }
+
+  /**
+   * Create verification runner with default configuration
+   * @param {Object} config - Configuration overrides
+   * @returns {VerificationRunner} - Runner instance
+   */
+  static createDefault(config = {}) {
+    return new VerificationRunner({
+      parallel: true,
+      maxConcurrency: 3,
+      timeout: 300000,
+      continueOnFailure: true,
+      ...config
+    });
+  }
+
+  /**
+   * Create verification runner for sequential execution
+   * @param {Object} config - Configuration overrides
+   * @returns {VerificationRunner} - Sequential runner instance
+   */
+  static createSequential(config = {}) {
+    return new VerificationRunner({
+      parallel: false,
+      maxConcurrency: 1,
+      timeout: 600000, // Longer timeout for sequential
+      continueOnFailure: false,
+      ...config
+    });
+  }
+
+  /**
+   * Get runner information
+   * @returns {Object} - Runner info
+   */
+  getRunnerInfo() {
+    return {
+      parallel: this.parallel,
+      maxConcurrency: this.maxConcurrency,
+      timeout: this.timeout,
+      continueOnFailure: this.continueOnFailure,
+      supportedTestTypes: [
+        VerificationType.FILE_OPERATION,
+        VerificationType.COMMAND_EXECUTION
+        // TODO: Add API_CALL and NETWORK_TEST when implemented
+      ],
+      features: [
+        'parallel execution',
+        'timeout protection',
+        'batch processing',
+        'error handling',
+        'statistics collection'
+      ],
+      limitations: [
+        'limited to implemented verifiers',
+        'memory usage with parallel tests',
+        'timeout affects all tests'
+      ]
+    };
+  }
+}
+
+module.exports = VerificationRunner;