qa360 2.1.7 → 2.2.0

package/core/src/pack-v2/validator.ts

@@ -0,0 +1,726 @@
+/**
+ * QA360 Pack v2 Validator
+ *
+ * Validates pack.yml v2 files with test_files patterns and auth profiles.
+ */
+
+import { glob } from 'glob';
+import { existsSync } from 'fs';
+import { resolve, dirname } from 'path';
+import {
+  PackConfigV2,
+  PackValidationResultV2,
+  ValidationError,
+  AuthConfigV2,
+  GateConfigV2,
+  AuthProfile,
+  AuthTypeV2,
+  Hook,
+  ExecutionConfigV2
+} from '../types/pack-v2.js';
+
+export class PackValidatorV2 {
+  private packPath: string;
+  private baseDir: string;
+
+  constructor(packPath: string) {
+    this.packPath = packPath;
+    this.baseDir = dirname(resolve(packPath));
+  }
+
+  /**
+   * Validate a Pack v2 configuration
+   */
+  async validate(pack: any, options?: { checkFilesExist?: boolean }): Promise<PackValidationResultV2> {
+    const errors: ValidationError[] = [];
+    const warnings: ValidationError[] = [];
+    const info: { totalTests?: number; gatesCount?: number; frameworks?: string[] } = {
+      gatesCount: 0,
+      totalTests: 0,
+      frameworks: []
+    };
+
+    // 1. Validate structure
+    const structureErrors = this.validateStructure(pack);
+    errors.push(...structureErrors);
+
+    if (errors.length > 0) {
+      return {
+        valid: false,
+        version: 2,
+        errors,
+        warnings,
+        info
+      };
+    }
+
+    // 2. Validate version
+    if (pack.version !== 2) {
+      errors.push({
+        code: 'QP2V001',
+        message: `Invalid version: expected 2, got ${pack.version}`,
+        path: 'version'
+      });
+      return { valid: false, version: 2, errors, warnings, info };
+    }
+
+    // 3. Validate required fields
+    const requiredErrors = this.validateRequiredFields(pack as PackConfigV2);
+    errors.push(...requiredErrors);
+
+    // 4. Validate fixtures configuration
+    const fixturesErrors = await this.validateFixtures(pack.fixtures);
+    errors.push(...fixturesErrors.errors);
+    warnings.push(...fixturesErrors.warnings);
+
+    // 5. Validate auth configuration
+    const authErrors = await this.validateAuthConfig(pack.auth);
+    errors.push(...authErrors.errors);
+    warnings.push(...authErrors.warnings);
+
+    // 6. Validate gates
+    const gatesResult = await this.validateGates(pack.gates, options?.checkFilesExist ?? false);
+    errors.push(...gatesResult.errors);
+    warnings.push(...gatesResult.warnings);
+    info.gatesCount = Object.keys(pack.gates || {}).length;
+    info.totalTests = gatesResult.totalTests;
+    info.frameworks = gatesResult.frameworks;
+
+    // 7. Validate hooks
+    const hooksResult = this.validateHooks(pack.hooks);
+    errors.push(...hooksResult.errors);
+    warnings.push(...hooksResult.warnings);
+
+    // 8. Validate execution config
+    const execErrors = this.validateExecutionConfig(pack.execution);
+    errors.push(...execErrors.errors);
+    warnings.push(...execErrors.warnings);
+
+    // 9. Business logic validation
+    const businessErrors = this.validateBusinessRules(pack as PackConfigV2);
+    errors.push(...businessErrors.errors);
+    warnings.push(...businessErrors.warnings);
+
+    return {
+      valid: errors.length === 0,
+      version: 2,
+      errors,
+      warnings,
+      info
+    };
+  }
+
+  /**
+   * Validate basic structure
+   */
+  private validateStructure(pack: any): ValidationError[] {
+    const errors: ValidationError[] = [];
+
+    if (!pack || typeof pack !== 'object') {
+      errors.push({
+        code: 'QP2V002',
+        message: 'Pack configuration must be an object',
+        path: 'root'
+      });
+    }
+
+    return errors;
+  }
+
+  /**
+   * Validate required fields
+   */
+  private validateRequiredFields(pack: PackConfigV2): ValidationError[] {
+    const errors: ValidationError[] = [];
+
+    if (!pack.name || typeof pack.name !== 'string') {
+      errors.push({
+        code: 'QP2V003',
+        message: 'Pack name is required and must be a string',
+        path: 'name',
+        suggestion: 'Add: name: "my-pack"'
+      });
+    }
+
+    if (!pack.gates || typeof pack.gates !== 'object') {
+      errors.push({
+        code: 'QP2V004',
+        message: 'Gates configuration is required and must be an object',
+        path: 'gates',
+        suggestion: 'Add: gates: { smoke: { adapter: "playwright-api", test_files: ["tests/**/*.spec.ts"] } }'
+      });
+    }
+
+    if (pack.gates && Object.keys(pack.gates).length === 0) {
+      errors.push({
+        code: 'QP2V005',
+        message: 'At least one gate must be defined',
+        path: 'gates',
+        suggestion: 'Add at least one gate configuration'
+      });
+    }
+
+    return errors;
+  }
+
+  /**
+   * Validate fixtures configuration
+   */
+  private async validateFixtures(fixtures?: string[]): Promise<{ errors: ValidationError[]; warnings: ValidationError[] }> {
+    const errors: ValidationError[] = [];
+    const warnings: ValidationError[] = [];
+
+    if (!fixtures || fixtures.length === 0) {
+      return { errors, warnings };
+    }
+
+    for (const fixturePath of fixtures) {
+      // Validate fixture path format
+      if (typeof fixturePath !== 'string') {
+        errors.push({
+          code: 'QP2V037',
+          message: `Fixture path must be a string: ${fixturePath}`,
+          path: 'fixtures',
+          suggestion: 'Use string paths for fixture files'
+        });
+        continue;
+      }
+
+      // Check for invalid patterns
+      if (fixturePath.includes('..')) {
+        errors.push({
+          code: 'QP2V038',
+          message: 'Fixture path cannot contain parent directory reference ".."',
+          path: `fixtures.${fixturePath}`,
+          suggestion: 'Use relative patterns without ..'
+        });
+      }
+
+      // Check file extension
+      const ext = fixturePath.split('.').pop()?.toLowerCase();
+      if (!['yml', 'yaml', 'json'].includes(ext || '')) {
+        warnings.push({
+          code: 'QP2V039',
+          message: `Fixture file "${fixturePath}" has unknown extension, expected .yml, .yaml, or .json`,
+          path: `fixtures.${fixturePath}`,
+          suggestion: 'Use .yml, .yaml, or .json extension for fixture files'
+        });
+      }
+    }
+
+    return { errors, warnings };
+  }
+
+  /**
+   * Validate auth configuration
+   */
+  private async validateAuthConfig(auth?: AuthConfigV2): Promise<{ errors: ValidationError[]; warnings: ValidationError[] }> {
+    const errors: ValidationError[] = [];
+    const warnings: ValidationError[] = [];
+
+    if (!auth) {
+      return { errors, warnings };
+    }
+
+    // Validate default auth profiles reference existing profiles
+    const profiles = auth.profiles || {};
+    const validAuthTypes: AuthTypeV2[] = ['none', 'jwt', 'oauth2', 'api_key', 'bearer', 'basic', 'totp', 'ui_login', 'gcp_adc', 'aws_iam', 'azure_ad'];
+
+    if (auth.api && !profiles[auth.api]) {
+      errors.push({
+        code: 'QP2V006',
+        message: `Auth profile "${auth.api}" referenced in auth.api but not defined in profiles`,
+        path: 'auth.api',
+        suggestion: `Define the profile or use an existing one: ${Object.keys(profiles).join(', ') || 'none defined'}`
+      });
+    }
+
+    if (auth.ui && !profiles[auth.ui]) {
+      errors.push({
+        code: 'QP2V007',
+        message: `Auth profile "${auth.ui}" referenced in auth.ui but not defined in profiles`,
+        path: 'auth.ui',
+        suggestion: `Define the profile or use an existing one: ${Object.keys(profiles).join(', ') || 'none defined'}`
+      });
+    }
+
+    // Validate each profile
+    for (const [name, profile] of Object.entries(profiles)) {
+      const profileErrors = this.validateAuthProfile(name, profile as AuthProfile, validAuthTypes);
+      errors.push(...profileErrors.errors);
+      warnings.push(...profileErrors.warnings);
+    }
+
+    return { errors, warnings };
+  }
+
+  /**
+   * Validate a single auth profile
+   */
+  private validateAuthProfile(name: string, profile: AuthProfile, validTypes: AuthTypeV2[]): { errors: ValidationError[]; warnings: ValidationError[] } {
+    const errors: ValidationError[] = [];
+    const warnings: ValidationError[] = [];
+
+    if (!validTypes.includes(profile.type)) {
+      errors.push({
+        code: 'QP2V008',
+        message: `Invalid auth type: ${profile.type}`,
+        path: `auth.profiles.${name}.type`,
+        suggestion: `Use one of: ${validTypes.join(', ')}`
+      });
+    }
+
+    // Type-specific validation
+    switch (profile.type) {
+      case 'jwt':
+        if (!profile.config?.client_id && !profile.config?.token_endpoint) {
+          warnings.push({
+            code: 'QP2V009',
+            message: 'JWT profile should have client_id or token_endpoint',
+            path: `auth.profiles.${name}.config`,
+            suggestion: 'Add client_id or token_endpoint for token retrieval'
+          });
+        }
+        break;
+
+      case 'oauth2':
+        if (!profile.config?.token_url) {
+          errors.push({
+            code: 'QP2V010',
+            message: 'OAuth2 profile requires token_url',
+            path: `auth.profiles.${name}.config.token_url`,
+            suggestion: 'Add token_url: "https://auth.example.com/oauth2/token"'
+          });
+        }
+        if (!profile.config?.client_id) {
+          errors.push({
+            code: 'QP2V011',
+            message: 'OAuth2 profile requires client_id',
+            path: `auth.profiles.${name}.config.client_id`,
+            suggestion: 'Add client_id: "your-client-id"'
+          });
+        }
+        break;
+
+      case 'api_key':
+        if (!profile.config?.key) {
+          errors.push({
+            code: 'QP2V012',
+            message: 'API key profile requires key',
+            path: `auth.profiles.${name}.config.key`,
+            suggestion: 'Add key or use secret reference: ${{ secrets.API_KEY }}'
+          });
+        }
+        break;
+
+      case 'bearer':
+        if (!profile.config?.token) {
+          errors.push({
+            code: 'QP2V013',
+            message: 'Bearer token profile requires token',
+            path: `auth.profiles.${name}.config.token`,
+            suggestion: 'Add token or use secret reference: ${{ secrets.BEARER_TOKEN }}'
+          });
+        }
+        break;
+
+      case 'basic':
+        if (!profile.config?.username || !profile.config?.password) {
+          warnings.push({
+            code: 'QP2V014',
+            message: 'Basic auth profile should have username and password',
+            path: `auth.profiles.${name}.config`,
+            suggestion: 'Add username and password or use secret references'
+          });
+        }
+        break;
+
+      case 'ui_login':
+        if (!profile.config?.url) {
+          errors.push({
+            code: 'QP2V015',
+            message: 'UI login profile requires url',
+            path: `auth.profiles.${name}.config.url`,
+            suggestion: 'Add url: "https://example.com/login"'
+          });
+        }
+        break;
+
+      case 'totp':
+        if (!profile.config?.secret) {
+          errors.push({
+            code: 'QP2V016',
+            message: 'TOTP profile requires secret',
+            path: `auth.profiles.${name}.config.secret`,
+            suggestion: 'Add TOTP secret or use secret reference'
+          });
+        }
+        break;
+    }
+
+    return { errors, warnings };
+  }
+
+  /**
+   * Validate gates configuration
+   */
+  private async validateGates(
+    gates: Record<string, GateConfigV2>,
+    checkFilesExist: boolean
+  ): Promise<{ errors: ValidationError[]; warnings: ValidationError[]; totalTests: number; frameworks: string[] }> {
+    const errors: ValidationError[] = [];
+    const warnings: ValidationError[] = [];
+    const frameworks = new Set<string>();
+    let totalTests = 0;
+
+    if (!gates) {
+      return { errors, warnings, totalTests, frameworks: [] };
+    }
+
+    const validAdapters = [
+      'playwright-api',
+      'playwright-ui',
+      'k6-perf',
+      'semgrep-sast',
+      'zap-dast',
+      'gitleaks-secrets',
+      'osv-deps'
+    ];
+
+    for (const [gateName, gate] of Object.entries(gates)) {
+      // Validate adapter
+      if (gate.adapter && !validAdapters.includes(gate.adapter)) {
+        warnings.push({
+          code: 'QP2V017',
+          message: `Unknown adapter: ${gate.adapter}`,
+          path: `gates.${gateName}.adapter`,
+          suggestion: `Use one of: ${validAdapters.join(', ')}`
+        });
+      }
+
+      // Collect adapter type
+      if (gate.adapter) {
+        frameworks.add(gate.adapter);
+      }
+
+      // Validate test_files
+      if (gate.test_files && gate.test_files.length > 0) {
+        for (const pattern of gate.test_files) {
+          const patternErrors = this.validateTestPattern(gateName, pattern, checkFilesExist);
+          errors.push(...patternErrors.errors);
+          warnings.push(...patternErrors.warnings);
+
+          // Count matching files
+          if (checkFilesExist) {
+            try {
+              const matches = await glob(pattern, { cwd: this.baseDir, absolute: false });
+              totalTests += matches.length;
+            } catch {
+              // Ignore glob errors
+            }
+          }
+        }
+      } else if (!gate.config && !gate.test_files) {
+        warnings.push({
+          code: 'QP2V018',
+          message: `Gate "${gateName}" has neither test_files nor config`,
+          path: `gates.${gateName}`,
+          suggestion: 'Add test_files: ["tests/**/*.spec.ts"] or config with inline configuration'
+        });
+      }
+
+      // Validate auth profile reference
+      if (gate.auth) {
+        // Will be validated against profiles in auth section
+        if (!gate.adapter?.includes('api') && !gate.adapter?.includes('ui')) {
+          warnings.push({
+            code: 'QP2V019',
+            message: `Auth profile specified but adapter may not support auth`,
+            path: `gates.${gateName}.auth`,
+            suggestion: 'Ensure adapter supports authentication'
+          });
+        }
+      }
+
+      // Validate budgets
+      if (gate.budgets) {
+        const budgetErrors = this.validateGateBudgets(gateName, gate.budgets);
+        errors.push(...budgetErrors);
+      }
+
+      // Validate dependencies
+      if (gate.depends_on) {
+        for (const dep of gate.depends_on) {
+          if (!gates[dep]) {
+            errors.push({
+              code: 'QP2V020',
+              message: `Gate depends on non-existent gate: ${dep}`,
+              path: `gates.${gateName}.depends_on`,
+              suggestion: `Remove ${dep} from dependencies or define the gate`
+            });
+          }
+        }
+      }
+    }
+
+    return { errors, warnings, totalTests, frameworks: Array.from(frameworks) };
+  }
+
+  /**
+   * Validate a test file pattern
+   */
+  private validateTestPattern(gateName: string, pattern: string, checkExistence: boolean): { errors: ValidationError[]; warnings: ValidationError[] } {
+    const errors: ValidationError[] = [];
+    const warnings: ValidationError[] = [];
+
+    // Check for invalid patterns
+    if (pattern.includes('..')) {
+      errors.push({
+        code: 'QP2V021',
+        message: 'Test pattern cannot contain parent directory reference ".."',
+        path: `gates.${gateName}.test_files`,
+        suggestion: 'Use relative patterns without ..'
+      });
+    }
+
+    if (pattern.startsWith('/')) {
+      warnings.push({
+        code: 'QP2V022',
+        message: 'Absolute path may not work across environments',
+        path: `gates.${gateName}.test_files`,
+        suggestion: 'Use relative pattern from pack file location'
+      });
+    }
+
+    return { errors, warnings };
+  }
+
+  /**
+   * Validate gate budgets
+   */
+  private validateGateBudgets(gateName: string, budgets: any): ValidationError[] {
+    const errors: ValidationError[] = [];
+
+    for (const [key, value] of Object.entries(budgets)) {
+      if (typeof value !== 'number' || value < 0) {
+        errors.push({
+          code: 'QP2V023',
+          message: `Budget value must be a positive number: ${key}`,
+          path: `gates.${gateName}.budgets.${key}`,
+          suggestion: 'Use a positive number value'
+        });
+      }
+    }
+
+    return errors;
+  }
+
+  /**
+   * Validate hooks configuration
+   */
+  private validateHooks(hooks?: any): { errors: ValidationError[]; warnings: ValidationError[] } {
+    const errors: ValidationError[] = [];
+    const warnings: ValidationError[] = [];
+
+    if (!hooks) {
+      return { errors, warnings };
+    }
+
+    const validHookTypes = ['run', 'wait_on', 'script', 'docker'];
+    const validPhases = ['beforeAll', 'afterAll', 'beforeEach', 'afterEach'];
+
+    for (const [phase, hookList] of Object.entries(hooks)) {
+      if (!validPhases.includes(phase)) {
+        warnings.push({
+          code: 'QP2V024',
+          message: `Unknown hook phase: ${phase}`,
+          path: `hooks.${phase}`,
+          suggestion: `Use one of: ${validPhases.join(', ')}`
+        });
+        continue;
+      }
+
+      if (!Array.isArray(hookList)) {
+        errors.push({
+          code: 'QP2V025',
+          message: `Hook phase must be an array: ${phase}`,
+          path: `hooks.${phase}`,
+          suggestion: 'Use array format: hooks: { beforeAll: [{ run: "command" }] }'
+        });
+        continue;
+      }
+
+      for (let i = 0; i < hookList.length; i++) {
+        const hook = hookList[i] as Hook;
+        const hookErrors = this.validateHook(phase, i, hook, validHookTypes);
+        errors.push(...hookErrors.errors);
+        warnings.push(...hookErrors.warnings);
+      }
+    }
+
+    return { errors, warnings };
+  }
+
+  /**
+   * Validate a single hook
+   */
+  private validateHook(phase: string, index: number, hook: Hook, validTypes: string[]): { errors: ValidationError[]; warnings: ValidationError[] } {
+    const errors: ValidationError[] = [];
+    const warnings: ValidationError[] = [];
+
+    if (!hook.type || !validTypes.includes(hook.type)) {
+      errors.push({
+        code: 'QP2V026',
+        message: `Invalid or missing hook type: ${hook.type}`,
+        path: `hooks.${phase}[${index}].type`,
+        suggestion: `Use one of: ${validTypes.join(', ')}`
+      });
+      return { errors, warnings };
+    }
+
+    // Type-specific validation
+    switch (hook.type) {
+      case 'run':
+      case 'script':
+        if (!hook.command) {
+          errors.push({
+            code: 'QP2V027',
+            message: `${hook.type} hook requires command`,
+            path: `hooks.${phase}[${index}].command`,
+            suggestion: 'Add command to execute'
+          });
+        }
+        break;
+
+      case 'wait_on':
+        if (!hook.wait_for?.resource) {
+          errors.push({
+            code: 'QP2V028',
+            message: 'wait_on hook requires resource',
+            path: `hooks.${phase}[${index}].wait_for.resource`,
+            suggestion: 'Add resource: "http://localhost:3000" or "tcp://localhost:8080"'
+          });
+        }
+        break;
+
+      case 'docker':
+        if (!hook.compose) {
+          warnings.push({
+            code: 'QP2V029',
+            message: 'Docker hook should have compose configuration',
+            path: `hooks.${phase}[${index}]`,
+            suggestion: 'Add compose: { services: ["app"] }'
+          });
+        }
+        break;
+    }
+
+    // Check for dangerous commands
+    if (hook.command) {
+      const dangerous = ['rm -rf', 'sudo rm', 'del /f', 'format', 'mkfs'];
+      if (dangerous.some(d => hook.command!.toLowerCase().includes(d))) {
+        warnings.push({
+          code: 'QP2V030',
+          message: 'Potentially dangerous command in hook',
+          path: `hooks.${phase}[${index}].command`,
+          suggestion: 'Review command for safety'
+        });
+      }
+    }
+
+    return { errors, warnings };
+  }
+
+  /**
+   * Validate execution configuration
+   */
+  private validateExecutionConfig(execution?: ExecutionConfigV2): { errors: ValidationError[]; warnings: ValidationError[] } {
+    const errors: ValidationError[] = [];
+    const warnings: ValidationError[] = [];
+
+    if (!execution) {
+      // Add default if missing
+      return { errors, warnings };
+    }
+
+    // Validate on_failure
+    if (execution.on_failure && !['stop', 'continue', 'proceed'].includes(execution.on_failure)) {
+      errors.push({
+        code: 'QP2V031',
+        message: `Invalid on_failure value: ${execution.on_failure}`,
+        path: 'execution.on_failure',
+        suggestion: 'Use one of: stop, continue, proceed'
+      });
+    }
+
+    // Validate numeric values
+    if (execution.default_timeout !== undefined && (typeof execution.default_timeout !== 'number' || execution.default_timeout <= 0)) {
+      errors.push({
+        code: 'QP2V032',
+        message: 'default_timeout must be a positive number',
+        path: 'execution.default_timeout',
+        suggestion: 'Use timeout in milliseconds (e.g., 30000 for 30s)'
+      });
+    }
+
+    if (execution.max_concurrency !== undefined && (typeof execution.max_concurrency !== 'number' || execution.max_concurrency <= 0)) {
+      errors.push({
+        code: 'QP2V033',
+        message: 'max_concurrency must be a positive number',
+        path: 'execution.max_concurrency',
+        suggestion: 'Use a value >= 1'
+      });
+    }
+
+    return { errors, warnings };
+  }
+
+  /**
+   * Validate business rules
+   */
+  private validateBusinessRules(pack: PackConfigV2): { errors: ValidationError[]; warnings: ValidationError[] } {
+    const errors: ValidationError[] = [];
+    const warnings: ValidationError[] = [];
+
+    // Early return if no gates (already caught by required validation)
+    if (!pack.gates || Object.keys(pack.gates).length === 0) {
+      return { errors, warnings };
+    }
+
+    // Check for API gate with api auth but no auth profile
+    const apiGates = Object.entries(pack.gates).filter(([_, g]) => g.adapter?.includes('api'));
+    if (apiGates.length > 0 && !pack.auth?.api && !pack.auth?.profiles) {
+      warnings.push({
+        code: 'QP2V034',
+        message: 'API gates defined but no default auth profile configured',
+        path: 'auth.api',
+        suggestion: 'Add auth.api or define profiles for API authentication'
+      });
+    }
+
+    // Check for UI gate with ui auth but no auth profile
+    const uiGates = Object.entries(pack.gates).filter(([_, g]) => g.adapter?.includes('ui'));
+    if (uiGates.length > 0 && !pack.auth?.ui && !pack.auth?.profiles) {
+      warnings.push({
+        code: 'QP2V035',
+        message: 'UI gates defined but no default auth profile configured',
+        path: 'auth.ui',
+        suggestion: 'Add auth.ui or define profiles for UI authentication'
+      });
+    }
+
+    // Check for gates without adapter
+    const gatesWithoutAdapter = Object.entries(pack.gates).filter(([_, g]) => !g.adapter);
+    if (gatesWithoutAdapter.length > 0) {
+      const names = gatesWithoutAdapter.map(([name]) => name).join(', ');
+      warnings.push({
+        code: 'QP2V036',
+        message: `Gates without adapter: ${names}`,
+        path: 'gates',
+        suggestion: 'Add adapter to each gate for proper execution'
+      });
+    }
+
+    return { errors, warnings };
+  }
+}