qa360 2.2.1 → 2.2.7

package/cli/dist/core/pack/validator.js

@@ -0,0 +1,322 @@
+/**
+ * QA360 Pack v1 Validator
+ * Validates pack.yml files against the official schema
+ */
+import Ajv from 'ajv';
+import addFormats from 'ajv-formats';
+import { readFileSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+export class PackValidator {
+    ajv;
+    schema;
+    constructor() {
+        this.ajv = new Ajv({
+            allErrors: true,
+            verbose: true,
+            strict: false
+        });
+        addFormats(this.ajv);
+        // Load schema (ES modules compatible)
+        const __filename = fileURLToPath(import.meta.url);
+        const __dirname = dirname(__filename);
+        const schemaPath = join(__dirname, '../../schemas/pack.schema.json');
+        this.schema = JSON.parse(readFileSync(schemaPath, 'utf8'));
+        this.ajv.addSchema(this.schema, 'pack-v1');
+    }
+    /**
+     * Validate a pack configuration
+     */
+    validate(pack) {
+        const errors = [];
+        const warnings = [];
+        // Check version first to determine which schema to use
+        const version = pack.version;
+        if (version === 2) {
+            // Version 2 uses a different structure (object gates)
+            // For now, return a helpful message that v2 is not fully supported
+            errors.push({
+                code: 'QP200',
+                path: 'version',
+                message: 'Pack version 2 is not yet supported in this release',
+                suggestion: 'Use version: 1 with the v1 format. See docs for v1 format examples.'
+            });
+            return {
+                valid: false,
+                errors,
+                warnings
+            };
+        }
+        if (version > 2 || version < 1) {
+            errors.push({
+                code: 'QP201',
+                path: 'version',
+                message: `Unsupported pack version: ${version}`,
+                suggestion: 'Use version: 1 (current) or version: 2 (coming soon)'
+            });
+            return {
+                valid: false,
+                errors,
+                warnings
+            };
+        }
+        // Schema validation for v1
+        const valid = this.ajv.validate('pack-v1', pack);
+        if (!valid && this.ajv.errors) {
+            for (const error of this.ajv.errors) {
+                errors.push({
+                    code: this.getErrorCode(error),
+                    path: error.instancePath || error.schemaPath || 'root',
+                    message: this.formatErrorMessage(error),
+                    suggestion: this.getSuggestion(error)
+                });
+            }
+        }
+        // Custom business logic validation
+        if (valid) {
+            const businessRules = this.validateBusinessRules(pack);
+            errors.push(...businessRules.errors);
+            warnings.push(...businessRules.warnings);
+        }
+        return {
+            valid: errors.length === 0,
+            errors,
+            warnings
+        };
+    }
+    /**
+     * Validate business rules beyond schema
+     */
+    validateBusinessRules(pack) {
+        const errors = [];
+        const warnings = [];
+        // Check gate-target consistency (only for non-minimal packs)
+        if (pack.gates.includes('api_smoke') && !pack.targets?.api) {
+            // For minimal smoke packs, this is a warning, not an error
+            if (pack.gates.length === 1 && pack.gates[0] === 'api_smoke') {
+                warnings.push({
+                    code: 'QP001',
+                    path: 'targets.api',
+                    message: 'API smoke gate without api target - using default configuration',
+                    suggestion: 'Add targets.api with baseUrl for better control'
+                });
+            }
+            else {
+                errors.push({
+                    code: 'QP001',
+                    path: 'targets.api',
+                    message: 'API smoke gate requires api target configuration',
+                    suggestion: 'Add targets.api with baseUrl'
+                });
+            }
+        }
+        if (pack.gates.includes('ui') && !pack.targets?.web) {
+            errors.push({
+                code: 'QP002',
+                path: 'targets.web',
+                message: 'UI gate requires web target configuration',
+                suggestion: 'Add targets.web with baseUrl'
+            });
+        }
+        // Check budget recommendations
+        if (pack.gates.includes('perf') && !pack.budgets?.perf_p95_ms) {
+            warnings.push({
+                code: 'QP003',
+                path: 'budgets.perf_p95_ms',
+                message: 'Performance gate without budget may not fail appropriately',
+                suggestion: 'Add budgets.perf_p95_ms (recommended: 800-2000ms)'
+            });
+        }
+        if (pack.gates.includes('a11y') && !pack.budgets?.a11y_min) {
+            warnings.push({
+                code: 'QP004',
+                path: 'budgets.a11y_min',
+                message: 'Accessibility gate without budget may not fail appropriately',
+                suggestion: 'Add budgets.a11y_min (recommended: 90-95%)'
+            });
+        }
+        // Check security configuration
+        if ((pack.gates.includes('sast') || pack.gates.includes('dast')) && !pack.security) {
+            warnings.push({
+                code: 'QP005',
+                path: 'security',
+                message: 'Security gates without security configuration',
+                suggestion: 'Add security section with sast_max_high and secrets_leak settings'
+            });
+        }
+        // Check for potential secrets in plain text (ignore secret references)
+        if (pack.environment) {
+            const SECRET_REF = /\$\{\{\s*secrets\.[A-Z0-9_]+\s*\}\}/g;
+            for (const [key, value] of Object.entries(pack.environment)) {
+                if (typeof value === 'string' && !SECRET_REF.test(value) && this.looksLikeSecret(value)) {
+                    errors.push({
+                        code: 'QP006',
+                        path: `environment.${key}`,
+                        message: 'Potential secret detected in plain text',
+                        suggestion: `Use secret reference: \${{ secrets.${key.toUpperCase()} }}`
+                    });
+                }
+            }
+        }
+        // Check for dangerous hook commands
+        if (pack.hooks) {
+            const dangerousCommands = ['rm -rf', 'sudo rm', 'del /f', 'format', 'mkfs', 'dd if='];
+            ['beforeAll', 'beforeEach', 'afterEach', 'afterAll'].forEach(hookType => {
+                const hooks = pack.hooks[hookType];
+                if (hooks) {
+                    hooks.forEach((hook, index) => {
+                        const command = typeof hook === 'string' ? hook :
+                            ('run' in hook ? hook.run :
+                                'compose' in hook ? `compose ${hook.compose}` :
+                                    'wait_on' in hook ? `wait_on ${hook.wait_on}` : '');
+                        if (command && dangerousCommands.some(dangerous => command.toLowerCase().includes(dangerous))) {
+                            warnings.push({
+                                code: 'QP007',
+                                path: `hooks.${hookType}[${index}]`,
+                                message: 'Potentially dangerous command detected',
+                                suggestion: 'Review command for safety'
+                            });
+                        }
+                    });
+                }
+            });
+        }
+        return { errors, warnings };
+    }
+    /**
+     * Check if a value looks like a secret
+     */
+    looksLikeSecret(value) {
+        const secretPatterns = [
+            /^[A-Za-z0-9+/]{20,}={0,2}$/, // Base64-like
+            /^[a-f0-9]{32,}$/i, // Hex tokens
+            /^sk-[a-zA-Z0-9]{32,}$/, // API keys
+            /^ghp_[a-zA-Z0-9]{36}$/, // GitHub tokens
+            /^xoxb-[a-zA-Z0-9-]+$/, // Slack tokens
+        ];
+        return secretPatterns.some(pattern => pattern.test(value));
+    }
+    /**
+     * Check if a value is a secret reference
+     */
+    isSecretReference(value) {
+        return /^\$\{\{\s*secrets\.[A-Z_][A-Z0-9_]*\s*\}\}$/.test(value);
+    }
+    /**
+     * Validate hook commands for security issues
+     */
+    validateHookSecurity(hooks, basePath, warnings) {
+        for (let i = 0; i < hooks.length; i++) {
+            const hook = hooks[i];
+            if (typeof hook.run === 'string') {
+                // Check for dangerous commands
+                const dangerousPatterns = [
+                    /rm\s+-rf\s+\//, // rm -rf /
+                    /sudo\s+/, // sudo commands
+                    /curl.*\|\s*sh/, // curl | sh
+                    /wget.*\|\s*sh/, // wget | sh
+                ];
+                for (const pattern of dangerousPatterns) {
+                    if (pattern.test(hook.run)) {
+                        warnings.push({
+                            code: 'QP007',
+                            path: `${basePath}[${i}].run`,
+                            message: 'Potentially dangerous command detected',
+                            suggestion: 'Review command for security implications'
+                        });
+                        break;
+                    }
+                }
+            }
+        }
+    }
+    /**
+     * Get error code from AJV error
+     */
+    getErrorCode(error) {
+        const codeMap = {
+            'required': 'QP100',
+            'type': 'QP101',
+            'format': 'QP102',
+            'pattern': 'QP103',
+            'enum': 'QP104',
+            'const': 'QP105',
+            'minimum': 'QP106',
+            'maximum': 'QP107',
+            'minLength': 'QP108',
+            'maxLength': 'QP109',
+            'minItems': 'QP110',
+            'uniqueItems': 'QP111',
+            'additionalProperties': 'QP112'
+        };
+        return codeMap[error.keyword] || 'QP999';
+    }
+    /**
+     * Format error message for better UX
+     */
+    formatErrorMessage(error) {
+        const path = error.instancePath || 'root';
+        switch (error.keyword) {
+            case 'required':
+                return `Missing required field: ${error.params.missingProperty}`;
+            case 'type':
+                return `Expected ${error.params.type}, got ${typeof error.data}`;
+            case 'format':
+                return `Invalid ${error.params.format} format`;
+            case 'pattern':
+                return `Value does not match required pattern`;
+            case 'enum':
+                return `Value must be one of: ${error.params.allowedValues.join(', ')}`;
+            case 'const':
+                return `Value must be exactly: ${error.params.allowedValue}`;
+            case 'minimum':
+                return `Value must be >= ${error.params.limit}`;
+            case 'maximum':
+                return `Value must be <= ${error.params.limit}`;
+            case 'minLength':
+                return `Value must be at least ${error.params.limit} characters`;
+            case 'maxLength':
+                return `Value must be at most ${error.params.limit} characters`;
+            case 'minItems':
+                return `Array must have at least ${error.params.limit} items`;
+            case 'uniqueItems':
+                return `Array items must be unique`;
+            case 'additionalProperties':
+                return `Unknown property: ${error.params.additionalProperty}`;
+            default:
+                return error.message || 'Validation error';
+        }
+    }
+    /**
+     * Get suggestion for fixing error
+     */
+    getSuggestion(error) {
+        switch (error.keyword) {
+            case 'required':
+                const field = error.params.missingProperty;
+                const suggestions = {
+                    'version': 'Add: version: 1',
+                    'name': 'Add: name: "my-pack"',
+                    'gates': 'Add: gates: ["api_smoke"]',
+                    'baseUrl': 'Add: baseUrl: "https://api.example.com"'
+                };
+                return suggestions[field];
+            case 'enum':
+                return `Use one of: ${error.params.allowedValues.join(', ')}`;
+            case 'format':
+                if (error.params.format === 'uri') {
+                    return 'Use full URL with protocol: https://example.com';
+                }
+                break;
+            case 'pattern':
+                if (error.instancePath.includes('name')) {
+                    return 'Use only letters, numbers, underscore, and hyphen';
+                }
+                if (error.instancePath.includes('smoke')) {
+                    return 'Format: "GET /path -> 200"';
+                }
+                break;
+        }
+        return undefined;
+    }
+}

package/cli/dist/core/pack-v2/index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * QA360 Pack v2 Module
+ *
+ * Provides validation, migration, and loading for Pack v2 configurations.
+ */
+export { PackValidatorV2 } from './validator.js';
+export { PackMigrator } from './migrator.js';
+export { PackLoaderV2, PackLoadResult } from './loader.js';
+export type { PackConfigV2, AuthConfigV2, AuthProfile, AuthTypeV2, AuthCacheConfig, GateConfigV2, GateBudgets, GateOptions, HooksConfig, Hook, ExecutionConfigV2, ResourceLimits, SecurityBudgets, PerformanceTargets, JWTAuthProfileV2, OAuth2AuthProfileV2, UILoginAuthProfileV2, GCPADCProfileV2, AWSIamProfileV2, TargetConfig, ApiTarget, WebTarget, ApiRequest, WebPage, PageAction, PackValidationResultV2, ValidationError, PackMigrationResultV2 } from '../types/pack-v2.js';

package/cli/dist/core/pack-v2/index.js

@@ -0,0 +1,8 @@
+/**
+ * QA360 Pack v2 Module
+ *
+ * Provides validation, migration, and loading for Pack v2 configurations.
+ */
+export { PackValidatorV2 } from './validator.js';
+export { PackMigrator } from './migrator.js';
+export { PackLoaderV2 } from './loader.js';

package/cli/dist/core/pack-v2/loader.d.ts

@@ -0,0 +1,63 @@
+/**
+ * QA360 Pack v2 Loader
+ *
+ * Loads and parses pack.yml v2 files from the filesystem.
+ * Supports environment variable substitution with ${VAR} and ${VAR:-default} syntax.
+ */
+import { PackConfigV2, PackValidationResultV2, ValidationError } from '../types/pack-v2.js';
+export interface PackLoadResult {
+    success: boolean;
+    pack?: PackConfigV2 | any;
+    validation?: PackValidationResultV2;
+    valid?: boolean;
+    error?: Error;
+    sourcePath?: string;
+    format?: 'v1' | 'v2' | 'unknown' | 'legacy';
+    migrated?: boolean;
+    changes?: string[];
+    warnings?: string[];
+    foundFiles?: string[];
+    missingFiles?: string[];
+    validationErrors?: ValidationError[];
+}
+export declare class PackLoaderV2 {
+    private cache;
+    /**
+     * Load a pack configuration from a file path
+     */
+    load(packPath: string, options?: {
+        validate?: boolean;
+        checkFilesExist?: boolean;
+        useCache?: boolean;
+        migrate?: boolean;
+    }): Promise<PackLoadResult>;
+    /**
+     * Load and validate in one step
+     */
+    loadAndValidate(packPath: string, options?: {
+        checkFilesExist?: boolean;
+    }): Promise<PackLoadResult>;
+    /**
+     * Load multiple packs from a directory
+     */
+    loadFromDir(dirPath: string, options?: {
+        pattern?: string;
+        validate?: boolean;
+        checkFilesExist?: boolean;
+    }): Promise<PackLoadResult[]>;
+    /**
+     * Load from a YAML string (useful for testing or dynamic configs)
+     */
+    loadFromString(yamlContent: string, options?: {
+        validate?: boolean;
+        filename?: string;
+    }): Promise<PackLoadResult>;
+    /**
+     * Clear the cache
+     */
+    clearCache(path?: string): void;
+    /**
+     * Get cache size
+     */
+    getCacheSize(): number;
+}

package/cli/dist/core/pack-v2/loader.js

@@ -0,0 +1,292 @@
+/**
+ * QA360 Pack v2 Loader
+ *
+ * Loads and parses pack.yml v2 files from the filesystem.
+ * Supports environment variable substitution with ${VAR} and ${VAR:-default} syntax.
+ */
+import { readFileSync, existsSync } from 'fs';
+import { resolve } from 'path';
+import * as yaml from 'js-yaml';
+import { PackValidatorV2 } from './validator.js';
+import { PackMigrator } from './migrator.js';
+/**
+ * Substitute environment variables in a string
+ * Supports ${VAR} and ${VAR:-default} syntax
+ */
+function substituteEnvVars(str) {
+    return str.replace(/\$\{([^}:]+)(:-([^}]*))?\}/g, (match, varName, _, defaultValue) => {
+        return process.env[varName] ?? defaultValue ?? match;
+    });
+}
+/**
+ * Recursively substitute environment variables in an object
+ */
+function substituteEnvVarsRecursive(obj) {
+    if (typeof obj === 'string') {
+        return substituteEnvVars(obj);
+    }
+    if (Array.isArray(obj)) {
+        return obj.map(substituteEnvVarsRecursive);
+    }
+    if (obj !== null && typeof obj === 'object') {
+        const result = {};
+        for (const [key, value] of Object.entries(obj)) {
+            result[key] = substituteEnvVarsRecursive(value);
+        }
+        return result;
+    }
+    return obj;
+}
+export class PackLoaderV2 {
+    cache = new Map();
+    /**
+     * Load a pack configuration from a file path
+     */
+    async load(packPath, options) {
+        try {
+            // Resolve absolute path
+            const resolvedPath = resolve(packPath);
+            // Check if file exists
+            if (!existsSync(resolvedPath)) {
+                return {
+                    success: false,
+                    error: new Error(`Pack file not found: ${resolvedPath}`),
+                    sourcePath: resolvedPath,
+                    format: 'unknown'
+                };
+            }
+            // Read file
+            const content = readFileSync(resolvedPath, 'utf-8');
+            // Parse YAML
+            const parsed = yaml.load(content);
+            // Substitute environment variables
+            const substituted = substituteEnvVarsRecursive(parsed);
+            // Detect version
+            const migrator = new PackMigrator();
+            const detectedVersion = migrator.detectVersion(substituted);
+            // Set format
+            const format = detectedVersion === 1 ? 'v1' : detectedVersion === 2 ? 'v2' : 'unknown';
+            // For v1 without migration
+            if (detectedVersion === 1 && options?.migrate === false) {
+                return {
+                    success: true,
+                    pack: substituted,
+                    format: 'v1',
+                    migrated: false,
+                    sourcePath: resolvedPath
+                };
+            }
+            let pack;
+            let migrated = false;
+            let changes = [];
+            let migrationWarnings = [];
+            if (detectedVersion === 1) {
+                // Check for legacy format before migrating
+                if (migrator.isLegacyFormat(substituted)) {
+                    return {
+                        success: false,
+                        error: new Error(`Legacy pack format detected (version ${substituted.version || '0.9.x'}). ` +
+                            `This format is no longer supported.\n\n` +
+                            `To fix this:\n` +
+                            `  1. Delete the old pack file: ${resolvedPath}\n` +
+                            `  2. Create a new pack: qa360 init\n` +
+                            `  3. Or copy an example: qa360 examples copy api-basic`),
+                        sourcePath: resolvedPath,
+                        format: 'legacy'
+                    };
+                }
+                // Migrate v1 to v2
+                const migrationResult = migrator.migrate(substituted);
+                if (!migrationResult.success) {
+                    return {
+                        success: false,
+                        error: new Error(`Migration failed: ${migrationResult.warnings.join(', ')}`),
+                        sourcePath: resolvedPath,
+                        format: 'v1'
+                    };
+                }
+                pack = migrationResult.migrated;
+                migrated = true;
+                changes = migrationResult.changes;
+                migrationWarnings = migrationResult.warnings;
+            }
+            else if (detectedVersion === 2) {
+                pack = substituted;
+            }
+            else {
+                // Check for legacy format and provide helpful error
+                if (migrator.isLegacyFormat(substituted)) {
+                    return {
+                        success: false,
+                        error: new Error(`Legacy pack format detected (version ${substituted.version || '0.9.x'}). ` +
+                            `This format is no longer supported.\n\n` +
+                            `To fix this:\n` +
+                            `  1. Delete the old pack file: ${resolvedPath}\n` +
+                            `  2. Create a new pack: qa360 init\n` +
+                            `  3. Or copy an example: qa360 examples copy api-basic`),
+                        sourcePath: resolvedPath,
+                        format: 'legacy'
+                    };
+                }
+                return {
+                    success: false,
+                    error: new Error(`Unknown pack format. Version: ${substituted.version}`),
+                    sourcePath: resolvedPath,
+                    format: 'unknown'
+                };
+            }
+            // Cache the pack
+            if (options?.useCache !== false) {
+                this.cache.set(resolvedPath, pack);
+            }
+            // Validate if requested (or by default)
+            const shouldValidate = options?.validate !== false;
+            if (shouldValidate) {
+                const validator = new PackValidatorV2(resolvedPath);
+                const validation = await validator.validate(pack, { checkFilesExist: options?.checkFilesExist });
+                return {
+                    success: validation.valid,
+                    pack,
+                    valid: validation.valid,
+                    validation,
+                    validationErrors: validation.errors,
+                    sourcePath: resolvedPath,
+                    format: 'v2',
+                    migrated,
+                    changes,
+                    warnings: migrationWarnings,
+                    foundFiles: validation.info?.totalTests ? [`found ${validation.info.totalTests} tests`] : undefined
+                };
+            }
+            return {
+                success: true,
+                pack,
+                valid: true,
+                sourcePath: resolvedPath,
+                format: 'v2',
+                migrated,
+                changes,
+                warnings: migrationWarnings
+            };
+        }
+        catch (error) {
+            return {
+                success: false,
+                error: error,
+                sourcePath: packPath,
+                format: 'unknown'
+            };
+        }
+    }
+    /**
+     * Load and validate in one step
+     */
+    async loadAndValidate(packPath, options) {
+        return this.load(packPath, {
+            validate: true,
+            checkFilesExist: options?.checkFilesExist,
+            useCache: false
+        });
+    }
+    /**
+     * Load multiple packs from a directory
+     */
+    async loadFromDir(dirPath, options) {
+        const { glob } = await import('glob');
+        const pattern = options?.pattern || '**/*.yml';
+        const results = [];
+        try {
+            const files = await glob(pattern, {
+                cwd: dirPath,
+                absolute: true,
+                ignore: ['**/node_modules/**']
+            });
+            for (const file of files) {
+                const result = await this.load(file, options);
+                results.push(result);
+            }
+            return results;
+        }
+        catch (error) {
+            return [{
+                    success: false,
+                    error: new Error(`Failed to scan directory: ${error.message}`)
+                }];
+        }
+    }
+    /**
+     * Load from a YAML string (useful for testing or dynamic configs)
+     */
+    async loadFromString(yamlContent, options) {
+        try {
+            const parsed = yaml.load(yamlContent);
+            // Substitute environment variables
+            const substituted = substituteEnvVarsRecursive(parsed);
+            // Detect version and migrate if needed
+            const migrator = new PackMigrator();
+            const detectedVersion = migrator.detectVersion(substituted);
+            let pack;
+            let migrated = false;
+            if (detectedVersion === 1) {
+                const migrationResult = migrator.migrate(substituted);
+                if (!migrationResult.success) {
+                    return {
+                        success: false,
+                        error: new Error(`Migration failed: ${migrationResult.warnings.join(', ')}`)
+                    };
+                }
+                pack = migrationResult.migrated;
+                migrated = true;
+            }
+            else if (detectedVersion === 2) {
+                pack = substituted;
+            }
+            else {
+                return {
+                    success: false,
+                    error: new Error(`Unknown pack format. Version: ${substituted.version}`)
+                };
+            }
+            // Validate if requested
+            if (options?.validate !== false) {
+                const fakePath = options?.filename || 'pack.yml';
+                const validator = new PackValidatorV2(fakePath);
+                const validation = await validator.validate(pack);
+                return {
+                    success: validation.valid,
+                    pack,
+                    validation,
+                    migrated
+                };
+            }
+            return {
+                success: true,
+                pack,
+                migrated
+            };
+        }
+        catch (error) {
+            return {
+                success: false,
+                error: new Error(`Failed to parse YAML: ${error.message}`)
+            };
+        }
+    }
+    /**
+     * Clear the cache
+     */
+    clearCache(path) {
+        if (path) {
+            this.cache.delete(resolve(path));
+        }
+        else {
+            this.cache.clear();
+        }
+    }
+    /**
+     * Get cache size
+     */
+    getCacheSize() {
+        return this.cache.size;
+    }
+}