vibecodingmachine-core 2026.2.20-438 → 2026.2.26-1642

package/src/agents/AgentVerificationService.js

@@ -0,0 +1,634 @@
+/**
+ * Agent Verification Service
+ * 
+ * Orchestrates agent verification across multiple verifiers.
+ * Follows constitutional requirements: <555 lines, test-first approach.
+ */
+
+const VerificationRunner = require('./verification/VerificationRunner');
+const AgentCommunicationTester = require('./verification/AgentCommunicationTester');
+const TestRequirementManager = require('./verification/TestRequirementManager');
+const ResultAnalyzer = require('./verification/ResultAnalyzer');
+const { VerificationType } = require('./VerificationType');
+const { OperationType, OperationStatus } = require('./OperationTypes');
+
+/**
+ * Agent verification service class
+ */
+class AgentVerificationService {
+  /**
+   * Create verification service instance
+   * @param {Object} config - Service configuration
+   */
+  constructor(config = {}) {
+    this.configManager = config.configManager || null;
+    this.logger = config.logger || null;
+    this.fileManager = config.fileManager || null;
+    this.parallel = config.parallel !== false; // Default to parallel
+    this.maxConcurrency = config.maxConcurrency || 3;
+    this.timeout = config.timeout || 180000; // 3 minutes default
+    this.continueOnFailure = config.continueOnFailure !== false; // Default to continue
+    this.skipVerification = config.skipVerification || false;
+    this.enableCommunicationTesting = config.enableCommunicationTesting !== false;
+    
+    // Initialize communication testing components
+    if (this.enableCommunicationTesting) {
+      this.testManager = new TestRequirementManager({
+        logger: this.logger,
+        ...config.testManagerOptions
+      });
+      
+      this.communicationTester = new AgentCommunicationTester({
+        testManager: this.testManager,
+        logger: this.logger,
+        timeout: this.timeout,
+        maxConcurrentTests: this.maxConcurrency,
+        continueOnFailure: this.continueOnFailure,
+        ...config.communicationTesterOptions
+      });
+      
+      this.resultAnalyzer = new ResultAnalyzer({
+        logger: this.logger,
+        detailedAnalysis: config.detailedAnalysis !== false,
+        ...config.analyzerOptions
+      });
+    }
+  }
+
+  /**
+   * Verify agent using available verifiers
+   * @param {Object} agent - Agent to verify
+   * @param {Object} options - Verification options
+   * @returns {Promise<Object>} - Verification result
+   */
+  async verifyAgent(agent, options = {}) {
+    const { skipVerification = this.skipVerification, testTypes = null, timeout = this.timeout } = options;
+    
+    if (skipVerification) {
+      return {
+        success: true,
+        agentId: agent.id,
+        skipped: true,
+        message: 'Verification skipped by configuration',
+        timestamp: new Date().toISOString()
+      };
+    }
+
+    if (this.logger) {
+      await this.logger.info('Starting agent verification', {
+        agentId: agent.id,
+        agentName: agent.name,
+        testTypes,
+        timeout
+      });
+    }
+
+    const startTime = Date.now();
+    
+    try {
+      // Update agent status to VERIFYING
+      await this.updateAgentStatus(agent.id, AgentStatus.VERIFYING, 'Starting verification');
+      
+      // Create verification test configurations
+      const testConfigs = this.createTestConfigs(agent, testTypes);
+      
+      if (testConfigs.length === 0) {
+        throw new Error('No verification tests configured for agent');
+      }
+
+      // Run verification tests
+      const runner = VerificationRunner.createDefault({
+        parallel: this.parallel,
+        maxConcurrency: this.maxConcurrency,
+        timeout,
+        continueOnFailure: this.continueOnFailure,
+        logger: this.logger
+      });
+      
+      const verificationResult = await runner.runVerificationTests(agent, testConfigs);
+      
+      // Run communication tests if enabled
+      let communicationResult = null;
+      if (this.enableCommunicationTesting) {
+        if (this.logger) {
+          await this.logger.info('Starting agent communication testing', {
+            agentId: agent.id,
+            agentName: agent.name
+          });
+        }
+        
+        communicationResult = await this.communicationTester.testCommunication(agent, {
+          testTypes: [VerificationType.FILE_OPERATION, VerificationType.COMMAND_EXECUTION]
+        });
+        
+        if (this.logger) {
+          await this.logger.info('Agent communication testing completed', {
+            agentId: agent.id,
+            success: communicationResult.success,
+            summary: communicationResult.summary
+          });
+        }
+      }
+      
+      // Combine results and determine overall status
+      const standardTestsPassed = verificationResult.summary.allPassed;
+      const communicationTestsPassed = !communicationResult || communicationResult.success;
+      const allPassed = standardTestsPassed && communicationTestsPassed;
+      
+      const finalStatus = allPassed ? AgentStatus.VERIFIED : AgentStatus.ERROR;
+      const message = allPassed ? 'All verification tests passed' : 'Some verification tests failed';
+      
+      // Analyze results if analyzer is available
+      let analysis = null;
+      if (this.resultAnalyzer) {
+        const combinedResult = {
+          success: allPassed,
+          agentId: agent.id,
+          results: [
+            ...verificationResult.results,
+            ...(communicationResult ? communicationResult.results : [])
+          ],
+          summary: {
+            ...verificationResult.summary,
+            communicationTests: communicationResult ? communicationResult.summary : null
+          }
+        };
+        
+        analysis = this.resultAnalyzer.analyzeAgentResult(combinedResult);
+      }
+      
+      // Update agent status
+      await this.updateAgentStatus(agent.id, finalStatus, message);
+      
+      const duration = Date.now() - startTime;
+      
+      // Log verification result
+      await this.logVerification(agent, verificationResult, duration);
+      
+      return {
+        success: allPassed,
+        agentId: agent.id,
+        skipped: false,
+        finalStatus,
+        message,
+        duration,
+        result: verificationResult,
+        communicationResult,
+        analysis,
+        timestamp: new Date().toISOString()
+      };
+    } catch (error) {
+      const duration = Date.now() - startTime;
+      
+      // Update agent status to ERROR
+      await this.updateAgentStatus(agent.id, AgentStatus.ERROR, error.message);
+      
+      // Log verification failure
+      await this.logVerificationFailure(agent, error, duration);
+      
+      return {
+        success: false,
+        agentId: agent.id,
+        skipped: false,
+        error: error.message,
+        duration,
+        timestamp: new Date().toISOString()
+      };
+    }
+  }
+
+  /**
+   * Verify multiple agents
+   * @param {Array<Object>} agents - Agents to verify
+   * @param {Object} options - Verification options
+   * @returns {Promise<Object>} - Batch verification result
+   */
+  async verifyAgents(agents, options = {}) {
+    if (this.logger) {
+      await this.logger.info('Starting batch agent verification', {
+        agentCount: agents.length,
+        parallel: this.parallel,
+        maxConcurrency: this.maxConcurrency
+      });
+    }
+
+    const startTime = Date.now();
+    const results = {
+      totalAgents: agents.length,
+      successful: 0,
+      failed: 0,
+      skipped: 0,
+      duration: 0,
+      results: {},
+      summary: {
+        allSuccessful: false,
+        hasFailures: false,
+        hasSkipped: false
+      }
+    };
+
+    try {
+      if (this.parallel) {
+        // Verify agents in parallel with concurrency limit
+        const batches = this.createBatches(agents, this.maxConcurrency);
+        
+        for (const batch of batches) {
+          const batchResults = await this.verifyBatchInParallel(batch, options);
+          this.mergeBatchResults(results, batchResults);
+        }
+      } else {
+        // Verify agents sequentially
+        for (const agent of agents) {
+          const result = await this.verifyAgent(agent, options);
+          this.addVerificationResult(results, result);
+        }
+      }
+    } catch (error) {
+      if (this.logger) {
+        await this.logger.error('Batch verification error', {
+          agentCount: agents.length,
+          error: error.message
+        });
+      }
+      
+      // Add error as failed verification for all agents
+      for (const agent of agents) {
+        const errorResult = {
+          success: false,
+          agentId: agent.id,
+          error: error.message,
+          duration: 0,
+          timestamp: new Date().toISOString()
+        };
+        
+        this.addVerificationResult(results, errorResult);
+      }
+    }
+
+    results.duration = Date.now() - startTime;
+    this.calculateSummary(results);
+
+    if (this.logger) {
+      await this.logger.info('Batch verification completed', {
+        totalAgents: results.totalAgents,
+        successful: results.successful,
+        failed: results.failed,
+        skipped: results.skipped,
+        duration: results.duration,
+        allSuccessful: results.summary.allSuccessful
+      });
+    }
+
+    return results;
+  }
+
+  /**
+   * Verify a batch of agents in parallel
+   * @param {Array<Object>} batch - Batch of agents
+   * @param {Object} options - Verification options
+   * @returns {Promise<Array<Object>>} - Batch results
+   */
+  async verifyBatchInParallel(batch, options) {
+    const promises = batch.map(agent => 
+      this.verifyAgent(agent, options)
+    );
+
+    return Promise.allSettled(promises);
+  }
+
+  /**
+   * Merge batch results into main results
+   * @param {Object} results - Main results
+   * @param {Array<Object>} batchResults - Batch results
+   */
+  mergeBatchResults(results, batchResults) {
+    for (const batchResult of batchResults) {
+      if (batchResult.status === 'fulfilled') {
+        this.addVerificationResult(results, batchResult.value);
+      } else {
+        // Handle rejected promise
+        const errorResult = {
+          success: false,
+          agentId: 'unknown',
+          error: batchResult.reason?.message || 'Unknown error',
+          duration: 0,
+          timestamp: new Date().toISOString()
+        };
+        
+        this.addVerificationResult(results, errorResult);
+      }
+    }
+  }
+
+  /**
+   * Add verification result to results collection
+   * @param {Object} results - Results collection
+   * @param {Object} result - New result to add
+   */
+  addVerificationResult(results, result) {
+    results.results[result.agentId] = result;
+    
+    if (result.success) {
+      results.successful++;
+    } else {
+      results.failed++;
+    }
+    
+    if (result.skipped) {
+      results.skipped++;
+    }
+  }
+
+  /**
+   * Calculate summary statistics
+   * @param {Object} results - Results collection
+   */
+  calculateSummary(results) {
+    results.summary = {
+      allSuccessful: results.failed === 0 && results.successful === results.totalAgents,
+      hasFailures: results.failed > 0,
+      hasSkipped: results.skipped > 0,
+      successRate: results.totalAgents > 0 ? (results.successful / results.totalAgents) * 100 : 0,
+      failureRate: results.totalAgents > 0 ? (results.failed / results.totalAgents) * 100 : 0
+    };
+  }
+
+  /**
+   * Create verification test configurations for an agent
+   * @param {Object} agent - Agent configuration
+   * @param {Array<string>} testTypes - Specific test types to run
+   * @returns {Array<Object>} - Test configurations
+   */
+  createTestConfigs(agent, testTypes = null) {
+    const testConfigs = [];
+    
+    // Use agent's verification commands if available
+    if (agent.verificationCommands && agent.verificationCommands.length > 0) {
+      for (const command of agent.verificationCommands) {
+        testConfigs.push({
+          type: VerificationType.COMMAND_EXECUTION,
+          description: `Verify ${command}`,
+          command: command,
+          expectedResult: 'version',
+          timeout: 15,
+          agentCommand: command
+        });
+      }
+    }
+
+    // Add file operation test
+    if (!testTypes || testTypes.includes(VerificationType.FILE_OPERATION)) {
+      testConfigs.push({
+        type: VerificationType.FILE_OPERATION,
+        description: 'File create/read/delete test',
+        command: 'echo "test-requirement-{timestamp}" > test-requirement-{timestamp}.txt && cat test-requirement-{timestamp}.txt && rm test-requirement-{timestamp}.txt',
+        expectedResult: 'test-requirement-{timestamp}',
+        timeout: 10
+      });
+    }
+
+    // Filter by requested test types
+    if (testTypes) {
+      return testConfigs.filter(config => testTypes.includes(config.type));
+    }
+
+    return testConfigs;
+  }
+
+  /**
+   * Create batches for parallel execution
+   * @param {Array<Object>} items - Items to batch
+   * @param {number} batchSize - Maximum batch size
+   * @returns {Array<Array<Object>>} - Array of batches
+   */
+  createBatches(items, batchSize) {
+    const batches = [];
+    
+    for (let i = 0; i < items.length; i += batchSize) {
+      batches.push(items.slice(i, i + batchSize));
+    }
+    
+    return batches;
+  }
+
+  /**
+   * Update agent status
+   * @param {string} agentId - Agent identifier
+   * @param {string} status - New status
+   * @param {string} message - Status message
+   */
+  async updateAgentStatus(agentId, status, message = '') {
+    if (this.configManager) {
+      await this.configManager.updateAgentStatus(agentId, status, message);
+    }
+  }
+
+  /**
+   * Log successful verification
+   * @param {Object} agent - Agent configuration
+   * @param {Object} result - Verification result
+   * @param {number} duration - Verification duration
+   */
+  async logVerification(agent, result, duration) {
+    if (this.fileManager) {
+      await this.fileManager.writeLog('verification', {
+        agentId: agent.id,
+        operation: OperationType.VERIFY,
+        status: result.summary.allPassed ? OperationStatus.COMPLETED : OperationStatus.FAILED,
+        message: `Verification ${result.summary.allPassed ? 'passed' : 'failed'} for ${agent.name}`,
+        duration,
+        result,
+        summary: result.summary,
+        timestamp: new Date().toISOString()
+      });
+    }
+  }
+
+  /**
+   * Log verification failure
+   * @param {Object} agent - Agent configuration
+   * @param {Error} error - Verification error
+   * @param {number} duration - Verification duration
+   */
+  async logVerificationFailure(agent, error, duration) {
+    if (this.fileManager) {
+      await this.fileManager.writeLog('verification', {
+        agentId: agent.id,
+        operation: OperationType.VERIFY,
+        status: OperationStatus.FAILED,
+        message: `Verification failed for ${agent.name}: ${error.message}`,
+        duration,
+        error: error.message,
+        timestamp: new Date().toISOString()
+      });
+    }
+  }
+
+  /**
+   * Get verification statistics for agents
+   * @param {Array<Object>} agents - Agents to get stats for
+   * @returns {Promise<Object>} - Verification statistics
+   */
+  async getVerificationStats(agents) {
+    const stats = {
+      totalAgents: agents.length,
+      verified: 0,
+      notVerified: 0,
+      error: 0,
+      disabled: 0,
+      lastVerified: null,
+      verificationRate: 0
+    };
+
+    for (const agent of agents) {
+      const status = await this.configManager.getAgentStatus(agent.id);
+      
+      if (status) {
+        switch (status.status) {
+          case AgentStatus.VERIFIED:
+            stats.verified++;
+            break;
+          case AgentStatus.ERROR:
+            stats.error++;
+            break;
+          case AgentStatus.DISABLED:
+            stats.disabled++;
+            break;
+          default:
+            stats.notVerified++;
+            break;
+        }
+        
+        // Track most recent verification
+        if (status.lastChecked) {
+          const lastChecked = new Date(status.lastChecked);
+          if (!stats.lastVerified || lastChecked > stats.lastVerified) {
+            stats.lastVerified = status.lastChecked;
+          }
+        }
+      } else {
+        stats.notVerified++;
+      }
+    }
+
+    // Calculate verification rate
+    const checkableAgents = stats.verified + stats.notVerified + stats.error;
+    stats.verificationRate = checkableAgents > 0 ? (stats.verified / checkableAgents) * 100 : 0;
+
+    return stats;
+  }
+
+  /**
+   * Cleanup old test requirement files
+   * @param {number} maxAge - Maximum age in milliseconds
+   * @returns {Promise<Object>} - Cleanup result
+   */
+  async cleanupTestRequirements(maxAge = 3600000) { // 1 hour default
+    if (!this.enableCommunicationTesting || !this.testManager) {
+      return {
+        success: false,
+        error: 'Communication testing not enabled'
+      };
+    }
+    
+    try {
+      const result = await this.testManager.cleanupOldFiles(maxAge);
+      
+      if (this.logger) {
+        await this.logger.info('Test requirement cleanup completed', {
+          maxAge,
+          cleanedCount: result.cleanedCount,
+          errorCount: result.errorCount
+        });
+      }
+      
+      return result;
+    } catch (error) {
+      if (this.logger) {
+        await this.logger.error('Test requirement cleanup failed', {
+          maxAge,
+          error: error.message
+        });
+      }
+      
+      return {
+        success: false,
+        error: error.message,
+        maxAge
+      };
+    }
+  }
+
+  /**
+   * Get verification service information
+   * @returns {Object} - Service info
+   */
+  getServiceInfo() {
+    return {
+      parallel: this.parallel,
+      maxConcurrency: this.maxConcurrency,
+      timeout: this.timeout,
+      continueOnFailure: this.continueOnFailure,
+      skipVerification: this.skipVerification,
+      enableCommunicationTesting: this.enableCommunicationTesting,
+      supportedTestTypes: [
+        VerificationType.FILE_OPERATION,
+        VerificationType.COMMAND_EXECUTION
+        // TODO: Add API_CALL and NETWORK_TEST when implemented
+      ],
+      features: [
+        'parallel verification',
+        'multiple test types',
+        'batch processing',
+        'error handling',
+        'status management',
+        'statistics collection',
+        'logging integration',
+        'communication testing',
+        'test requirement management',
+        'result analysis',
+        'automated cleanup'
+      ],
+      limitations: [
+        'limited to implemented verifiers',
+        'depends on agent configuration',
+        'timeout affects all tests',
+        'memory usage with parallel tests'
+      ]
+    };
+  }
+
+  /**
+   * Create verification service with default configuration
+   * @param {Object} config - Configuration overrides
+   * @returns {AgentVerificationService} - Service instance
+   */
+  static createDefault(config = {}) {
+    return new AgentVerificationService({
+      parallel: true,
+      maxConcurrency: 3,
+      timeout: 180000,
+      continueOnFailure: true,
+      skipVerification: false,
+      ...config
+    });
+  }
+
+  /**
+   * Create verification service for sequential execution
+   * @param {Object} config - Configuration overrides
+   * @returns {AgentVerificationService} - Sequential service instance
+   */
+  static createSequential(config = {}) {
+    return new AgentVerificationService({
+      parallel: false,
+      maxConcurrency: 1,
+      timeout: 300000, // Longer timeout for sequential
+      continueOnFailure: false,
+      skipVerification: false,
+      ...config
+    });
+  }
+}
+
+module.exports = AgentVerificationService;