qa360 2.2.1 → 2.2.13

package/core/src/pack-v2/validator.ts

@@ -1,731 +0,0 @@
-/**
- * QA360 Pack v2 Validator
- *
- * Validates pack.yml v2 files with test_files patterns and auth profiles.
- */
-
-import { glob } from 'glob';
-import { existsSync } from 'fs';
-import { resolve, dirname } from 'path';
-import {
-  PackConfigV2,
-  PackValidationResultV2,
-  ValidationError,
-  AuthConfigV2,
-  GateConfigV2,
-  AuthProfile,
-  AuthTypeV2,
-  Hook,
-  ExecutionConfigV2
-} from '../types/pack-v2.js';
-
-export class PackValidatorV2 {
-  private packPath: string;
-  private baseDir: string;
-
-  constructor(packPath: string) {
-    this.packPath = packPath;
-    this.baseDir = dirname(resolve(packPath));
-  }
-
-  /**
-   * Validate a Pack v2 configuration
-   */
-  async validate(pack: any, options?: { checkFilesExist?: boolean }): Promise<PackValidationResultV2> {
-    const errors: ValidationError[] = [];
-    const warnings: ValidationError[] = [];
-    const info: { totalTests?: number; gatesCount?: number; frameworks?: string[] } = {
-      gatesCount: 0,
-      totalTests: 0,
-      frameworks: []
-    };
-
-    // 1. Validate structure
-    const structureErrors = this.validateStructure(pack);
-    errors.push(...structureErrors);
-
-    if (errors.length > 0) {
-      return {
-        valid: false,
-        version: 2,
-        errors,
-        warnings,
-        info
-      };
-    }
-
-    // 2. Validate version
-    if (pack.version !== 2) {
-      errors.push({
-        code: 'QP2V001',
-        message: `Invalid version: expected 2, got ${pack.version}`,
-        path: 'version'
-      });
-      return { valid: false, version: 2, errors, warnings, info };
-    }
-
-    // 3. Validate required fields
-    const requiredErrors = this.validateRequiredFields(pack as PackConfigV2);
-    errors.push(...requiredErrors);
-
-    // 4. Validate fixtures configuration
-    const fixturesErrors = await this.validateFixtures(pack.fixtures);
-    errors.push(...fixturesErrors.errors);
-    warnings.push(...fixturesErrors.warnings);
-
-    // 5. Validate auth configuration
-    const authErrors = await this.validateAuthConfig(pack.auth);
-    errors.push(...authErrors.errors);
-    warnings.push(...authErrors.warnings);
-
-    // 6. Validate gates
-    const gatesResult = await this.validateGates(pack.gates, options?.checkFilesExist ?? false);
-    errors.push(...gatesResult.errors);
-    warnings.push(...gatesResult.warnings);
-    info.gatesCount = Object.keys(pack.gates || {}).length;
-    info.totalTests = gatesResult.totalTests;
-    info.frameworks = gatesResult.frameworks;
-
-    // 7. Validate hooks
-    const hooksResult = this.validateHooks(pack.hooks);
-    errors.push(...hooksResult.errors);
-    warnings.push(...hooksResult.warnings);
-
-    // 8. Validate execution config
-    const execErrors = this.validateExecutionConfig(pack.execution);
-    errors.push(...execErrors.errors);
-    warnings.push(...execErrors.warnings);
-
-    // 9. Business logic validation
-    const businessErrors = this.validateBusinessRules(pack as PackConfigV2);
-    errors.push(...businessErrors.errors);
-    warnings.push(...businessErrors.warnings);
-
-    return {
-      valid: errors.length === 0,
-      version: 2,
-      errors,
-      warnings,
-      info
-    };
-  }
-
-  /**
-   * Validate basic structure
-   */
-  private validateStructure(pack: any): ValidationError[] {
-    const errors: ValidationError[] = [];
-
-    if (!pack || typeof pack !== 'object') {
-      errors.push({
-        code: 'QP2V002',
-        message: 'Pack configuration must be an object',
-        path: 'root'
-      });
-    }
-
-    return errors;
-  }
-
-  /**
-   * Validate required fields
-   */
-  private validateRequiredFields(pack: PackConfigV2): ValidationError[] {
-    const errors: ValidationError[] = [];
-
-    if (!pack.name || typeof pack.name !== 'string') {
-      errors.push({
-        code: 'QP2V003',
-        message: 'Pack name is required and must be a string',
-        path: 'name',
-        suggestion: 'Add: name: "my-pack"'
-      });
-    }
-
-    if (!pack.gates || typeof pack.gates !== 'object') {
-      errors.push({
-        code: 'QP2V004',
-        message: 'Gates configuration is required and must be an object',
-        path: 'gates',
-        suggestion: 'Add: gates: { smoke: { adapter: "playwright-api", test_files: ["tests/**/*.spec.ts"] } }'
-      });
-    }
-
-    if (pack.gates && Object.keys(pack.gates).length === 0) {
-      errors.push({
-        code: 'QP2V005',
-        message: 'At least one gate must be defined',
-        path: 'gates',
-        suggestion: 'Add at least one gate configuration'
-      });
-    }
-
-    return errors;
-  }
-
-  /**
-   * Validate fixtures configuration
-   */
-  private async validateFixtures(fixtures?: string[]): Promise<{ errors: ValidationError[]; warnings: ValidationError[] }> {
-    const errors: ValidationError[] = [];
-    const warnings: ValidationError[] = [];
-
-    if (!fixtures || fixtures.length === 0) {
-      return { errors, warnings };
-    }
-
-    for (const fixturePath of fixtures) {
-      // Validate fixture path format
-      if (typeof fixturePath !== 'string') {
-        errors.push({
-          code: 'QP2V037',
-          message: `Fixture path must be a string: ${fixturePath}`,
-          path: 'fixtures',
-          suggestion: 'Use string paths for fixture files'
-        });
-        continue;
-      }
-
-      // Check for patterns that might cause issues
-      if (fixturePath.includes('..')) {
-        // Changed from error to warning - parent directory references are sometimes needed
-        warnings.push({
-          code: 'QP2V038',
-          message: `Fixture path contains parent directory reference "..": ${fixturePath}`,
-          path: `fixtures.${fixturePath}`,
-          suggestion: 'Consider using absolute paths or placing fixtures in the same directory as the pack'
-        });
-      }
-
-      // Check file extension
-      const ext = fixturePath.split('.').pop()?.toLowerCase();
-      if (!['yml', 'yaml', 'json'].includes(ext || '')) {
-        warnings.push({
-          code: 'QP2V039',
-          message: `Fixture file "${fixturePath}" has unknown extension, expected .yml, .yaml, or .json`,
-          path: `fixtures.${fixturePath}`,
-          suggestion: 'Use .yml, .yaml, or .json extension for fixture files'
-        });
-      }
-    }
-
-    return { errors, warnings };
-  }
-
-  /**
-   * Validate auth configuration
-   */
-  private async validateAuthConfig(auth?: AuthConfigV2): Promise<{ errors: ValidationError[]; warnings: ValidationError[] }> {
-    const errors: ValidationError[] = [];
-    const warnings: ValidationError[] = [];
-
-    if (!auth) {
-      return { errors, warnings };
-    }
-
-    // Validate default auth profiles reference existing profiles
-    const profiles = auth.profiles || {};
-    const validAuthTypes: AuthTypeV2[] = ['none', 'jwt', 'oauth2', 'api_key', 'bearer', 'basic', 'totp', 'ui_login', 'gcp_adc', 'aws_iam', 'azure_ad'];
-
-    if (auth.api && !profiles[auth.api]) {
-      errors.push({
-        code: 'QP2V006',
-        message: `Auth profile "${auth.api}" referenced in auth.api but not defined in profiles`,
-        path: 'auth.api',
-        suggestion: `Define the profile or use an existing one: ${Object.keys(profiles).join(', ') || 'none defined'}`
-      });
-    }
-
-    if (auth.ui && !profiles[auth.ui]) {
-      errors.push({
-        code: 'QP2V007',
-        message: `Auth profile "${auth.ui}" referenced in auth.ui but not defined in profiles`,
-        path: 'auth.ui',
-        suggestion: `Define the profile or use an existing one: ${Object.keys(profiles).join(', ') || 'none defined'}`
-      });
-    }
-
-    // Validate each profile
-    for (const [name, profile] of Object.entries(profiles)) {
-      const profileErrors = this.validateAuthProfile(name, profile as AuthProfile, validAuthTypes);
-      errors.push(...profileErrors.errors);
-      warnings.push(...profileErrors.warnings);
-    }
-
-    return { errors, warnings };
-  }
-
-  /**
-   * Validate a single auth profile
-   */
-  private validateAuthProfile(name: string, profile: AuthProfile, validTypes: AuthTypeV2[]): { errors: ValidationError[]; warnings: ValidationError[] } {
-    const errors: ValidationError[] = [];
-    const warnings: ValidationError[] = [];
-
-    if (!validTypes.includes(profile.type)) {
-      errors.push({
-        code: 'QP2V008',
-        message: `Invalid auth type: ${profile.type}`,
-        path: `auth.profiles.${name}.type`,
-        suggestion: `Use one of: ${validTypes.join(', ')}`
-      });
-    }
-
-    // Type-specific validation
-    switch (profile.type) {
-      case 'jwt':
-        if (!profile.config?.client_id && !profile.config?.token_endpoint) {
-          warnings.push({
-            code: 'QP2V009',
-            message: 'JWT profile should have client_id or token_endpoint',
-            path: `auth.profiles.${name}.config`,
-            suggestion: 'Add client_id or token_endpoint for token retrieval'
-          });
-        }
-        break;
-
-      case 'oauth2':
-        if (!profile.config?.token_url) {
-          errors.push({
-            code: 'QP2V010',
-            message: 'OAuth2 profile requires token_url',
-            path: `auth.profiles.${name}.config.token_url`,
-            suggestion: 'Add token_url: "https://auth.example.com/oauth2/token"'
-          });
-        }
-        if (!profile.config?.client_id) {
-          errors.push({
-            code: 'QP2V011',
-            message: 'OAuth2 profile requires client_id',
-            path: `auth.profiles.${name}.config.client_id`,
-            suggestion: 'Add client_id: "your-client-id"'
-          });
-        }
-        break;
-
-      case 'api_key':
-        if (!profile.config?.key) {
-          errors.push({
-            code: 'QP2V012',
-            message: 'API key profile requires key',
-            path: `auth.profiles.${name}.config.key`,
-            suggestion: 'Add key or use secret reference: ${{ secrets.API_KEY }}'
-          });
-        }
-        break;
-
-      case 'bearer':
-        if (!profile.config?.token) {
-          errors.push({
-            code: 'QP2V013',
-            message: 'Bearer token profile requires token',
-            path: `auth.profiles.${name}.config.token`,
-            suggestion: 'Add token or use secret reference: ${{ secrets.BEARER_TOKEN }}'
-          });
-        }
-        break;
-
-      case 'basic':
-        if (!profile.config?.username || !profile.config?.password) {
-          warnings.push({
-            code: 'QP2V014',
-            message: 'Basic auth profile should have username and password',
-            path: `auth.profiles.${name}.config`,
-            suggestion: 'Add username and password or use secret references'
-          });
-        }
-        break;
-
-      case 'ui_login':
-        if (!profile.config?.url) {
-          errors.push({
-            code: 'QP2V015',
-            message: 'UI login profile requires url',
-            path: `auth.profiles.${name}.config.url`,
-            suggestion: 'Add url: "https://example.com/login"'
-          });
-        }
-        break;
-
-      case 'totp':
-        if (!profile.config?.secret) {
-          errors.push({
-            code: 'QP2V016',
-            message: 'TOTP profile requires secret',
-            path: `auth.profiles.${name}.config.secret`,
-            suggestion: 'Add TOTP secret or use secret reference'
-          });
-        }
-        break;
-    }
-
-    return { errors, warnings };
-  }
-
-  /**
-   * Validate gates configuration
-   */
-  private async validateGates(
-    gates: Record<string, GateConfigV2>,
-    checkFilesExist: boolean
-  ): Promise<{ errors: ValidationError[]; warnings: ValidationError[]; totalTests: number; frameworks: string[] }> {
-    const errors: ValidationError[] = [];
-    const warnings: ValidationError[] = [];
-    const frameworks = new Set<string>();
-    let totalTests = 0;
-
-    if (!gates) {
-      return { errors, warnings, totalTests, frameworks: [] };
-    }
-
-    const validAdapters = [
-      'playwright-api',
-      'playwright-ui',
-      'k6-perf',
-      'semgrep-sast',
-      'zap-dast',
-      'gitleaks-secrets',
-      'osv-deps',
-      // v2.2.0 Unit Test Adapters
-      'vitest',
-      'jest',
-      'pytest'
-    ];
-
-    for (const [gateName, gate] of Object.entries(gates)) {
-      // Validate adapter
-      if (gate.adapter && !validAdapters.includes(gate.adapter)) {
-        warnings.push({
-          code: 'QP2V017',
-          message: `Unknown adapter: ${gate.adapter}`,
-          path: `gates.${gateName}.adapter`,
-          suggestion: `Use one of: ${validAdapters.join(', ')}`
-        });
-      }
-
-      // Collect adapter type
-      if (gate.adapter) {
-        frameworks.add(gate.adapter);
-      }
-
-      // Validate test_files
-      if (gate.test_files && gate.test_files.length > 0) {
-        for (const pattern of gate.test_files) {
-          const patternErrors = this.validateTestPattern(gateName, pattern, checkFilesExist);
-          errors.push(...patternErrors.errors);
-          warnings.push(...patternErrors.warnings);
-
-          // Count matching files
-          if (checkFilesExist) {
-            try {
-              const matches = await glob(pattern, { cwd: this.baseDir, absolute: false });
-              totalTests += matches.length;
-            } catch {
-              // Ignore glob errors
-            }
-          }
-        }
-      } else if (!gate.config && !gate.test_files) {
-        warnings.push({
-          code: 'QP2V018',
-          message: `Gate "${gateName}" has neither test_files nor config`,
-          path: `gates.${gateName}`,
-          suggestion: 'Add test_files: ["tests/**/*.spec.ts"] or config with inline configuration'
-        });
-      }
-
-      // Validate auth profile reference
-      if (gate.auth) {
-        // Will be validated against profiles in auth section
-        if (!gate.adapter?.includes('api') && !gate.adapter?.includes('ui')) {
-          warnings.push({
-            code: 'QP2V019',
-            message: `Auth profile specified but adapter may not support auth`,
-            path: `gates.${gateName}.auth`,
-            suggestion: 'Ensure adapter supports authentication'
-          });
-        }
-      }
-
-      // Validate budgets
-      if (gate.budgets) {
-        const budgetErrors = this.validateGateBudgets(gateName, gate.budgets);
-        errors.push(...budgetErrors);
-      }
-
-      // Validate dependencies
-      if (gate.depends_on) {
-        for (const dep of gate.depends_on) {
-          if (!gates[dep]) {
-            errors.push({
-              code: 'QP2V020',
-              message: `Gate depends on non-existent gate: ${dep}`,
-              path: `gates.${gateName}.depends_on`,
-              suggestion: `Remove ${dep} from dependencies or define the gate`
-            });
-          }
-        }
-      }
-    }
-
-    return { errors, warnings, totalTests, frameworks: Array.from(frameworks) };
-  }
-
-  /**
-   * Validate a test file pattern
-   */
-  private validateTestPattern(gateName: string, pattern: string, checkExistence: boolean): { errors: ValidationError[]; warnings: ValidationError[] } {
-    const errors: ValidationError[] = [];
-    const warnings: ValidationError[] = [];
-
-    // Check for invalid patterns
-    if (pattern.includes('..')) {
-      errors.push({
-        code: 'QP2V021',
-        message: 'Test pattern cannot contain parent directory reference ".."',
-        path: `gates.${gateName}.test_files`,
-        suggestion: 'Use relative patterns without ..'
-      });
-    }
-
-    if (pattern.startsWith('/')) {
-      warnings.push({
-        code: 'QP2V022',
-        message: 'Absolute path may not work across environments',
-        path: `gates.${gateName}.test_files`,
-        suggestion: 'Use relative pattern from pack file location'
-      });
-    }
-
-    return { errors, warnings };
-  }
-
-  /**
-   * Validate gate budgets
-   */
-  private validateGateBudgets(gateName: string, budgets: any): ValidationError[] {
-    const errors: ValidationError[] = [];
-
-    for (const [key, value] of Object.entries(budgets)) {
-      if (typeof value !== 'number' || value < 0) {
-        errors.push({
-          code: 'QP2V023',
-          message: `Budget value must be a positive number: ${key}`,
-          path: `gates.${gateName}.budgets.${key}`,
-          suggestion: 'Use a positive number value'
-        });
-      }
-    }
-
-    return errors;
-  }
-
-  /**
-   * Validate hooks configuration
-   */
-  private validateHooks(hooks?: any): { errors: ValidationError[]; warnings: ValidationError[] } {
-    const errors: ValidationError[] = [];
-    const warnings: ValidationError[] = [];
-
-    if (!hooks) {
-      return { errors, warnings };
-    }
-
-    const validHookTypes = ['run', 'wait_on', 'script', 'docker'];
-    const validPhases = ['beforeAll', 'afterAll', 'beforeEach', 'afterEach'];
-
-    for (const [phase, hookList] of Object.entries(hooks)) {
-      if (!validPhases.includes(phase)) {
-        warnings.push({
-          code: 'QP2V024',
-          message: `Unknown hook phase: ${phase}`,
-          path: `hooks.${phase}`,
-          suggestion: `Use one of: ${validPhases.join(', ')}`
-        });
-        continue;
-      }
-
-      if (!Array.isArray(hookList)) {
-        errors.push({
-          code: 'QP2V025',
-          message: `Hook phase must be an array: ${phase}`,
-          path: `hooks.${phase}`,
-          suggestion: 'Use array format: hooks: { beforeAll: [{ run: "command" }] }'
-        });
-        continue;
-      }
-
-      for (let i = 0; i < hookList.length; i++) {
-        const hook = hookList[i] as Hook;
-        const hookErrors = this.validateHook(phase, i, hook, validHookTypes);
-        errors.push(...hookErrors.errors);
-        warnings.push(...hookErrors.warnings);
-      }
-    }
-
-    return { errors, warnings };
-  }
-
-  /**
-   * Validate a single hook
-   */
-  private validateHook(phase: string, index: number, hook: Hook, validTypes: string[]): { errors: ValidationError[]; warnings: ValidationError[] } {
-    const errors: ValidationError[] = [];
-    const warnings: ValidationError[] = [];
-
-    if (!hook.type || !validTypes.includes(hook.type)) {
-      errors.push({
-        code: 'QP2V026',
-        message: `Invalid or missing hook type: ${hook.type}`,
-        path: `hooks.${phase}[${index}].type`,
-        suggestion: `Use one of: ${validTypes.join(', ')}`
-      });
-      return { errors, warnings };
-    }
-
-    // Type-specific validation
-    switch (hook.type) {
-      case 'run':
-      case 'script':
-        if (!hook.command) {
-          errors.push({
-            code: 'QP2V027',
-            message: `${hook.type} hook requires command`,
-            path: `hooks.${phase}[${index}].command`,
-            suggestion: 'Add command to execute'
-          });
-        }
-        break;
-
-      case 'wait_on':
-        if (!hook.wait_for?.resource) {
-          errors.push({
-            code: 'QP2V028',
-            message: 'wait_on hook requires resource',
-            path: `hooks.${phase}[${index}].wait_for.resource`,
-            suggestion: 'Add resource: "http://localhost:3000" or "tcp://localhost:8080"'
-          });
-        }
-        break;
-
-      case 'docker':
-        if (!hook.compose) {
-          warnings.push({
-            code: 'QP2V029',
-            message: 'Docker hook should have compose configuration',
-            path: `hooks.${phase}[${index}]`,
-            suggestion: 'Add compose: { services: ["app"] }'
-          });
-        }
-        break;
-    }
-
-    // Check for dangerous commands
-    if (hook.command) {
-      const dangerous = ['rm -rf', 'sudo rm', 'del /f', 'format', 'mkfs'];
-      if (dangerous.some(d => hook.command!.toLowerCase().includes(d))) {
-        warnings.push({
-          code: 'QP2V030',
-          message: 'Potentially dangerous command in hook',
-          path: `hooks.${phase}[${index}].command`,
-          suggestion: 'Review command for safety'
-        });
-      }
-    }
-
-    return { errors, warnings };
-  }
-
-  /**
-   * Validate execution configuration
-   */
-  private validateExecutionConfig(execution?: ExecutionConfigV2): { errors: ValidationError[]; warnings: ValidationError[] } {
-    const errors: ValidationError[] = [];
-    const warnings: ValidationError[] = [];
-
-    if (!execution) {
-      // Add default if missing
-      return { errors, warnings };
-    }
-
-    // Validate on_failure
-    if (execution.on_failure && !['stop', 'continue', 'proceed'].includes(execution.on_failure)) {
-      errors.push({
-        code: 'QP2V031',
-        message: `Invalid on_failure value: ${execution.on_failure}`,
-        path: 'execution.on_failure',
-        suggestion: 'Use one of: stop, continue, proceed'
-      });
-    }
-
-    // Validate numeric values
-    if (execution.default_timeout !== undefined && (typeof execution.default_timeout !== 'number' || execution.default_timeout <= 0)) {
-      errors.push({
-        code: 'QP2V032',
-        message: 'default_timeout must be a positive number',
-        path: 'execution.default_timeout',
-        suggestion: 'Use timeout in milliseconds (e.g., 30000 for 30s)'
-      });
-    }
-
-    if (execution.max_concurrency !== undefined && (typeof execution.max_concurrency !== 'number' || execution.max_concurrency <= 0)) {
-      errors.push({
-        code: 'QP2V033',
-        message: 'max_concurrency must be a positive number',
-        path: 'execution.max_concurrency',
-        suggestion: 'Use a value >= 1'
-      });
-    }
-
-    return { errors, warnings };
-  }
-
-  /**
-   * Validate business rules
-   */
-  private validateBusinessRules(pack: PackConfigV2): { errors: ValidationError[]; warnings: ValidationError[] } {
-    const errors: ValidationError[] = [];
-    const warnings: ValidationError[] = [];
-
-    // Early return if no gates (already caught by required validation)
-    if (!pack.gates || Object.keys(pack.gates).length === 0) {
-      return { errors, warnings };
-    }
-
-    // Check for API gate with api auth but no auth profile
-    const apiGates = Object.entries(pack.gates).filter(([_, g]) => g.adapter?.includes('api'));
-    if (apiGates.length > 0 && !pack.auth?.api && !pack.auth?.profiles) {
-      warnings.push({
-        code: 'QP2V034',
-        message: 'API gates defined but no default auth profile configured',
-        path: 'auth.api',
-        suggestion: 'Add auth.api or define profiles for API authentication'
-      });
-    }
-
-    // Check for UI gate with ui auth but no auth profile
-    const uiGates = Object.entries(pack.gates).filter(([_, g]) => g.adapter?.includes('ui'));
-    if (uiGates.length > 0 && !pack.auth?.ui && !pack.auth?.profiles) {
-      warnings.push({
-        code: 'QP2V035',
-        message: 'UI gates defined but no default auth profile configured',
-        path: 'auth.ui',
-        suggestion: 'Add auth.ui or define profiles for UI authentication'
-      });
-    }
-
-    // Check for gates without adapter
-    const gatesWithoutAdapter = Object.entries(pack.gates).filter(([_, g]) => !g.adapter);
-    if (gatesWithoutAdapter.length > 0) {
-      const names = gatesWithoutAdapter.map(([name]) => name).join(', ');
-      warnings.push({
-        code: 'QP2V036',
-        message: `Gates without adapter: ${names}`,
-        path: 'gates',
-        suggestion: 'Add adapter to each gate for proper execution'
-      });
-    }
-
-    return { errors, warnings };
-  }
-}