qa360 2.1.7 → 2.2.0

package/core/src/watch/watch-mode.ts

@@ -0,0 +1,565 @@
+/**
+ * QA360 Watch Mode
+ *
+ * Continuous testing mode that monitors file changes and re-runs tests.
+ * Features:
+ * - File watching with debounce
+ * - Smart test selection based on changed files
+ * - Benchmark mode for performance comparison
+ * - Statistics aggregation across runs
+ *
+ * @example
+ * ```typescript
+ * const watcher = new WatchMode({
+ *   workingDir: process.cwd(),
+ *   debounceMs: 300,
+ *   onRunStart: () => console.log('Running tests...'),
+ *   onRunComplete: (result) => console.log('Done:', result)
+ * });
+ *
+ * await watcher.start();
+ * ```
+ */
+
+import { watch, type FSWatcher } from 'fs';
+import { relative, join } from 'path';
+import { Phase3Runner, type Phase3RunResult } from '../runner/phase3-runner.js';
+import { PackConfigV2 } from '../types/pack-v2.js';
+import { readFile } from 'fs/promises';
+
+/**
+ * Watch mode configuration options
+ */
+export interface WatchModeOptions {
+  /** Working directory for tests */
+  workingDir: string;
+  /** Path to pack.yml file */
+  packPath?: string;
+  /** Debounce delay in milliseconds (default: 300) */
+  debounceMs?: number;
+  /** Patterns to ignore (default: node_modules, .git, dist) */
+  ignore?: string[];
+  /** Whether to run initial test on start (default: true) */
+  runOnStart?: boolean;
+  /** Whether to clear screen between runs (default: true) */
+  clearScreen?: boolean;
+  /** Callback when a run starts */
+  onRunStart?: (runNumber: number) => void;
+  /** Callback when a run completes */
+  onRunComplete?: (result: WatchRunResult) => void;
+  /** Callback when a file changes */
+  onFileChange?: (path: string) => void;
+  /** Callback on error */
+  onError?: (error: Error) => void;
+}
+
+/**
+ * Result of a single watch run
+ */
+export interface WatchRunResult {
+  /** Run number (incrementing) */
+  runNumber: number;
+  /** Timestamp when run started */
+  startedAt: Date;
+  /** Timestamp when run completed */
+  completedAt: Date;
+  /** Duration in milliseconds */
+  duration: number;
+  /** Phase3Runner result */
+  result: Phase3RunResult;
+  /** Files that triggered this run */
+  changedFiles: string[];
+  /** Whether this run was successful */
+  success: boolean;
+}
+
+/**
+ * Aggregated statistics across watch runs
+ */
+export interface WatchStats {
+  /** Total number of runs */
+  totalRuns: number;
+  /** Number of successful runs */
+  successfulRuns: number;
+  /** Number of failed runs */
+  failedRuns: number;
+  /** Average duration in milliseconds */
+  avgDuration: number;
+  /** Fastest run duration */
+  fastestRun: number;
+  /** Slowest run duration */
+  slowestRun: number;
+  /** Total duration of all runs */
+  totalDuration: number;
+  /** Trust score average */
+  avgTrustScore: number;
+  /** Total tests executed */
+  totalTests: number;
+  /** Total passed tests */
+  totalPassed: number;
+  /** Total failed tests */
+  totalFailed: number;
+}
+
+/**
+ * Benchmark configuration
+ */
+export interface BenchmarkConfig {
+  /** Number of iterations to run */
+  iterations: number;
+  /** Warm-up iterations (not counted in stats) */
+  warmupIterations: number;
+  /** Whether to run with cache enabled */
+  withCache: boolean;
+  /** Whether to run in parallel */
+  parallel?: boolean;
+}
+
+/**
+ * Benchmark result
+ */
+export interface BenchmarkResult {
+  /** Average duration across iterations */
+  avgDuration: number;
+  /** Minimum duration */
+  minDuration: number;
+  /** Maximum duration */
+  maxDuration: number;
+  /** Standard deviation */
+  stdDev: number;
+  /** Median duration */
+  median: number;
+  /** P95 duration */
+  p95: number;
+  /** P99 duration */
+  p99: number;
+  /** Number of successful runs */
+  successfulRuns: number;
+  /** Total iterations */
+  totalIterations: number;
+  /** Throughput (runs per second) */
+  throughput: number;
+}
+
+/**
+ * Watch Mode Manager
+ *
+ * Monitors file changes and runs tests automatically.
+ */
+export class WatchMode {
+  private options: Required<WatchModeOptions>;
+  private watchers: FSWatcher[] = [];
+  private runNumber = 0;
+  private isRunning = false;
+  private debounceTimer?: NodeJS.Timeout;
+  private pendingChanges = new Set<string>();
+  private runResults: WatchRunResult[] = [];
+  private stats: WatchStats = {
+    totalRuns: 0,
+    successfulRuns: 0,
+    failedRuns: 0,
+    avgDuration: 0,
+    fastestRun: Infinity,
+    slowestRun: 0,
+    totalDuration: 0,
+    avgTrustScore: 0,
+    totalTests: 0,
+    totalPassed: 0,
+    totalFailed: 0
+  };
+  private packConfig?: PackConfigV2;
+
+  constructor(options: WatchModeOptions) {
+    this.options = {
+      workingDir: options.workingDir,
+      packPath: options.packPath ?? join(options.workingDir, 'pack.yml'),
+      debounceMs: options.debounceMs ?? 300,
+      ignore: options.ignore ?? ['**/node_modules/**', '**/.git/**', '**/dist/**', '**/.qa360/**', '**/build/**'],
+      runOnStart: options.runOnStart ?? true,
+      clearScreen: options.clearScreen ?? true,
+      onRunStart: options.onRunStart ?? (() => {}),
+      onRunComplete: options.onRunComplete ?? (() => {}),
+      onFileChange: options.onFileChange ?? (() => {}),
+      onError: options.onError ?? ((err) => console.error('Watch error:', err))
+    };
+  }
+
+  /**
+   * Start watching files and running tests
+   */
+  async start(): Promise<void> {
+    if (this.isRunning) {
+      throw new Error('Watch mode is already running');
+    }
+
+    this.isRunning = true;
+
+    try {
+      // Load pack configuration
+      await this.loadPackConfig();
+
+      // Setup file watcher
+      this.setupWatcher();
+
+      // Run initial test if configured
+      if (this.options.runOnStart) {
+        await this.runTests([]);
+      }
+
+      this.log('Watch mode started. Press Ctrl+C to stop.');
+
+    } catch (error) {
+      this.isRunning = false;
+      throw error;
+    }
+  }
+
+  /**
+   * Stop watching files
+   */
+  async stop(): Promise<void> {
+    if (!this.isRunning) {
+      return;
+    }
+
+    this.isRunning = false;
+
+    if (this.debounceTimer) {
+      clearTimeout(this.debounceTimer);
+    }
+
+    // Close all file watchers
+    for (const watcher of this.watchers) {
+      watcher.close();
+    }
+    this.watchers = [];
+
+    this.log('Watch mode stopped.');
+    this.log(this.formatStats());
+  }
+
+  /**
+   * Get current statistics
+   */
+  getStats(): WatchStats {
+    return { ...this.stats };
+  }
+
+  /**
+   * Get run history
+   */
+  getRunHistory(): WatchRunResult[] {
+    return [...this.runResults];
+  }
+
+  /**
+   * Run benchmark tests
+   */
+  async benchmark(config: BenchmarkConfig): Promise<BenchmarkResult> {
+    this.log(`Starting benchmark: ${config.iterations} iterations (${config.warmupIterations} warmup)`);
+
+    const durations: number[] = [];
+    let successful = 0;
+
+    // Warmup runs
+    this.log('Warming up...');
+    for (let i = 0; i < config.warmupIterations; i++) {
+      await this.runTests([], { silent: true });
+    }
+
+    // Benchmark runs
+    this.log('Running benchmark...');
+    for (let i = 0; i < config.iterations; i++) {
+      const result = await this.runTests([], { silent: true });
+      durations.push(result.duration);
+      if (result.success) {
+        successful++;
+      }
+
+      // Progress indicator
+      if ((i + 1) % Math.max(1, Math.floor(config.iterations / 10)) === 0) {
+        this.log(`  Progress: ${i + 1}/${config.iterations}`);
+      }
+    }
+
+    return this.calculateBenchmarkStats(durations, successful, config.iterations);
+  }
+
+  /**
+   * Load pack configuration
+   */
+  private async loadPackConfig(): Promise<void> {
+    try {
+      const content = await readFile(this.options.packPath!, 'utf-8');
+      // Parse YAML (simple version, assumes YAML is preprocessed or uses JSON-compatible format)
+      // For full YAML support, you'd need a YAML parser
+      this.packConfig = JSON.parse(content);
+    } catch (error) {
+      // If YAML parsing fails, try to use a simple config
+      this.packConfig = {
+        version: 2,
+        name: 'watch-tests',
+        gates: {} as Record<string, any>,
+        execution: { on_failure: 'continue' as const }
+      };
+    }
+  }
+
+  /**
+   * Setup file watcher using native Node.js fs.watch
+   */
+  private setupWatcher(): void {
+    const watchPath = this.options.workingDir;
+
+    try {
+      const watcher = watch(watchPath, { recursive: true }, (eventType, filename) => {
+        if (filename) {
+          this.handleFileChange(join(watchPath, filename));
+        }
+      });
+
+      watcher.on('error', (error) => this.options.onError(error));
+      this.watchers.push(watcher);
+
+    } catch (error) {
+      this.options.onError(error as Error);
+    }
+  }
+
+  /**
+   * Check if a path should be ignored
+   */
+  private shouldIgnore(path: string): boolean {
+    const normalizedPath = path.replace(/\\/g, '/');
+
+    for (const pattern of this.options.ignore) {
+      const regexPattern = pattern
+        .replace(/\*\*/g, '.*')
+        .replace(/\*/g, '[^/]*')
+        .replace(/\//g, '\\/');
+
+      try {
+        const regex = new RegExp(regexPattern);
+        if (regex.test(normalizedPath)) {
+          return true;
+        }
+      } catch {
+        // Invalid regex pattern, skip
+      }
+    }
+
+    return false;
+  }
+
+  /**
+   * Handle file change with debouncing
+   */
+  private handleFileChange(path: string): void {
+    if (this.shouldIgnore(path)) {
+      return;
+    }
+
+    this.pendingChanges.add(path);
+    this.options.onFileChange(path);
+
+    if (this.debounceTimer) {
+      clearTimeout(this.debounceTimer);
+    }
+
+    this.debounceTimer = setTimeout(() => {
+      const changes = Array.from(this.pendingChanges);
+      this.pendingChanges.clear();
+      this.runTests(changes);
+    }, this.options.debounceMs);
+  }
+
+  /**
+   * Run tests and update statistics
+   */
+  private async runTests(changedFiles: string[], options: { silent?: boolean } = {}): Promise<WatchRunResult> {
+    const runNumber = ++this.runNumber;
+    const startedAt = new Date();
+
+    if (!options.silent) {
+      if (this.options.clearScreen) {
+        console.clear();
+      }
+      this.options.onRunStart(runNumber);
+    }
+
+    let result: WatchRunResult;
+
+    try {
+      const runner = new Phase3Runner({
+        workingDir: this.options.workingDir,
+        pack: this.packConfig!
+      });
+
+      const phase3Result = await runner.run();
+      const completedAt = new Date();
+      const duration = completedAt.getTime() - startedAt.getTime();
+      const success = phase3Result.success;
+
+      result = {
+        runNumber,
+        startedAt,
+        completedAt,
+        duration,
+        result: phase3Result,
+        changedFiles,
+        success
+      };
+
+      this.runResults.push(result);
+      this.updateStats(result);
+
+      if (!options.silent) {
+        this.logRunResult(result);
+      }
+
+    } catch (error) {
+      const completedAt = new Date();
+      const duration = completedAt.getTime() - startedAt.getTime();
+
+      result = {
+        runNumber,
+        startedAt,
+        completedAt,
+        duration,
+        result: {
+          success: false,
+          pack: this.packConfig!,
+          duration: 0,
+          gates: [],
+          hooks: { beforeAll: [], beforeEach: [], afterEach: [], afterAll: [] },
+          summary: { total: 0, passed: 0, failed: 1, trustScore: 0 },
+          error: error instanceof Error ? error.message : String(error)
+        } as Phase3RunResult,
+        changedFiles,
+        success: false
+      };
+
+      this.runResults.push(result);
+      this.updateStats(result);
+    }
+
+    this.options.onRunComplete(result);
+    return result;
+  }
+
+  /**
+   * Update aggregated statistics
+   */
+  private updateStats(result: WatchRunResult): void {
+    this.stats.totalRuns++;
+    this.stats.totalDuration += result.duration;
+
+    if (result.success) {
+      this.stats.successfulRuns++;
+      this.stats.avgTrustScore = (
+        (this.stats.avgTrustScore * (this.stats.successfulRuns - 1) + result.result.summary.trustScore) /
+        this.stats.successfulRuns
+      );
+    } else {
+      this.stats.failedRuns++;
+    }
+
+    this.stats.avgDuration = this.stats.totalDuration / this.stats.totalRuns;
+    this.stats.fastestRun = Math.min(this.stats.fastestRun, result.duration);
+    this.stats.slowestRun = Math.max(this.stats.slowestRun, result.duration);
+
+    this.stats.totalTests += result.result.summary.total;
+    this.stats.totalPassed += result.result.summary.passed;
+    this.stats.totalFailed += result.result.summary.failed;
+  }
+
+  /**
+   * Calculate benchmark statistics
+   */
+  private calculateBenchmarkStats(durations: number[], successful: number, total: number): BenchmarkResult {
+    const sorted = [...durations].sort((a, b) => a - b);
+    const sum = durations.reduce((a, b) => a + b, 0);
+    const avg = sum / durations.length;
+    const variance = durations.reduce((acc, val) => acc + Math.pow(val - avg, 2), 0) / durations.length;
+    const stdDev = Math.sqrt(variance);
+
+    return {
+      avgDuration: avg,
+      minDuration: sorted[0] ?? 0,
+      maxDuration: sorted[sorted.length - 1] ?? 0,
+      stdDev,
+      median: sorted[Math.floor(sorted.length / 2)] ?? 0,
+      p95: sorted[Math.floor(sorted.length * 0.95)] ?? 0,
+      p99: sorted[Math.floor(sorted.length * 0.99)] ?? 0,
+      successfulRuns: successful,
+      totalIterations: total,
+      throughput: 1000 / avg // runs per second
+    };
+  }
+
+  /**
+   * Log formatted run result
+   */
+  private logRunResult(result: WatchRunResult): void {
+    const { result: r, changedFiles, duration } = result;
+
+    console.log(`\n${result.success ? '✅' : '❌'} Run #${result.runNumber} (${duration}ms)`);
+
+    if (changedFiles.length > 0) {
+      console.log(`   Changed files:`);
+      for (const file of changedFiles) {
+        console.log(`     - ${relative(this.options.workingDir, file)}`);
+      }
+    }
+
+    console.log(`   Tests: ${r.summary.passed}/${r.summary.total} passed`);
+    console.log(`   Trust Score: ${r.summary.trustScore}`);
+    console.log(`   Gates: ${r.gates.length}`);
+
+    if (r.gates.length > 0) {
+      for (const gate of r.gates) {
+        console.log(`     ${gate.success ? '✅' : '❌'} ${gate.gate}`);
+      }
+    }
+
+    console.log(`\n${this.formatStats()}`);
+  }
+
+  /**
+   * Format statistics for display
+   */
+  private formatStats(): string {
+    const s = this.stats;
+    if (s.totalRuns === 0) {
+      return 'No runs yet';
+    }
+
+    const successRate = ((s.successfulRuns / s.totalRuns) * 100).toFixed(1);
+
+    return [
+      '',
+      '📊 Statistics:',
+      `  Runs: ${s.totalRuns} (${s.successfulRuns} passed, ${s.failedRuns} failed)`,
+      `  Success Rate: ${successRate}%`,
+      `  Avg Duration: ${Math.round(s.avgDuration)}ms`,
+      `  Fastest: ${s.fastestRun === Infinity ? 0 : s.fastestRun}ms | Slowest: ${s.slowestRun}ms`,
+      `  Avg Trust Score: ${Math.round(s.avgTrustScore)}`,
+      `  Total Tests: ${s.totalTests} (${s.totalPassed} passed, ${s.totalFailed} failed)`,
+      ''
+    ].join('\n');
+  }
+
+  /**
+   * Log message
+   */
+  private log(message: string): void {
+    console.log(`[Watch] ${message}`);
+  }
+}
+
+/**
+ * Factory function to create a watch mode instance
+ */
+export function createWatchMode(options: WatchModeOptions): WatchMode {
+  return new WatchMode(options);
+}

package/core/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "extends": "../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "dist",
+    "rootDir": "src",
+    "composite": true,
+    "declaration": true,
+    "declarationMap": false
+  },
+  "include": ["src/**/*"],
+  "exclude": ["src/**/__tests__/**", "src/**/*.test.ts", "src/**/*.spec.ts"]
+}

package/core/vitest.config.ts

@@ -0,0 +1,52 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    globals: true,
+    environment: 'node',
+    pool: 'forks',
+    poolOptions: {
+      forks: {
+        singleFork: true,
+      },
+    },
+    bail: 0,
+    include: ['src/**/*.{test,spec}.{js,ts}', 'src/**/__tests__/**/*.{js,ts}'],
+    exclude: [
+      '**/node_modules/**',
+      '**/dist/**',
+      '**/*.d.ts',
+      'src/proof/__tests__/schema-validation-manual.mjs',
+      // E2E tests (Playwright/Mocha) - not for vitest
+      'tests/**',
+      'tests/e2e/**',
+      '**/tests/**',
+      '.tmp-test-*/**',
+      // Real provider integration tests (require API keys)
+      'src/generation/__tests__/real-provider-integration.test.ts',
+      // WIP - experimental features not yet implemented
+      'src/assertions/__tests__/**',
+      'src/crawler/__tests__/**',
+      'src/generation/__tests__/crawler-pack-generator.test.ts',
+      'src/adapters/__tests__/playwright-ui-e2e.test.ts',
+    ],
+    coverage: {
+      provider: 'v8',
+      reporter: ['text', 'json', 'html', 'lcov'],
+      include: ['src/**/*.ts'],
+      exclude: [
+        '**/*.d.ts',
+        '**/*.test.ts',
+        '**/*.spec.ts',
+        '**/__tests__/**',
+        '**/node_modules/**',
+        '**/dist/**',
+      ],
+      all: true,
+      lines: 20,
+      functions: 20,
+      branches: 20,
+      statements: 20,
+    },
+  },
+});