qa360 1.4.5 → 2.0.1

package/dist/core/pack-v2/validator.js

@@ -0,0 +1,577 @@
+/**
+ * QA360 Pack v2 Validator
+ *
+ * Validates pack.yml v2 files with test_files patterns and auth profiles.
+ */
+import { glob } from 'glob';
+import { resolve, dirname } from 'path';
+export class PackValidatorV2 {
+    packPath;
+    baseDir;
+    constructor(packPath) {
+        this.packPath = packPath;
+        this.baseDir = dirname(resolve(packPath));
+    }
+    /**
+     * Validate a Pack v2 configuration
+     */
+    async validate(pack, options) {
+        const errors = [];
+        const warnings = [];
+        const info = {
+            gatesCount: 0,
+            totalTests: 0,
+            frameworks: []
+        };
+        // 1. Validate structure
+        const structureErrors = this.validateStructure(pack);
+        errors.push(...structureErrors);
+        if (errors.length > 0) {
+            return {
+                valid: false,
+                version: 2,
+                errors,
+                warnings,
+                info
+            };
+        }
+        // 2. Validate version
+        if (pack.version !== 2) {
+            errors.push({
+                code: 'QP2V001',
+                message: `Invalid version: expected 2, got ${pack.version}`,
+                path: 'version'
+            });
+            return { valid: false, version: 2, errors, warnings, info };
+        }
+        // 3. Validate required fields
+        const requiredErrors = this.validateRequiredFields(pack);
+        errors.push(...requiredErrors);
+        // 4. Validate auth configuration
+        const authErrors = await this.validateAuthConfig(pack.auth);
+        errors.push(...authErrors.errors);
+        warnings.push(...authErrors.warnings);
+        // 5. Validate gates
+        const gatesResult = await this.validateGates(pack.gates, options?.checkFilesExist ?? false);
+        errors.push(...gatesResult.errors);
+        warnings.push(...gatesResult.warnings);
+        info.gatesCount = Object.keys(pack.gates || {}).length;
+        info.totalTests = gatesResult.totalTests;
+        info.frameworks = gatesResult.frameworks;
+        // 6. Validate hooks
+        const hooksResult = this.validateHooks(pack.hooks);
+        errors.push(...hooksResult.errors);
+        warnings.push(...hooksResult.warnings);
+        // 7. Validate execution config
+        const execErrors = this.validateExecutionConfig(pack.execution);
+        errors.push(...execErrors.errors);
+        warnings.push(...execErrors.warnings);
+        // 8. Business logic validation
+        const businessErrors = this.validateBusinessRules(pack);
+        errors.push(...businessErrors.errors);
+        warnings.push(...businessErrors.warnings);
+        return {
+            valid: errors.length === 0,
+            version: 2,
+            errors,
+            warnings,
+            info
+        };
+    }
+    /**
+     * Validate basic structure
+     */
+    validateStructure(pack) {
+        const errors = [];
+        if (!pack || typeof pack !== 'object') {
+            errors.push({
+                code: 'QP2V002',
+                message: 'Pack configuration must be an object',
+                path: 'root'
+            });
+        }
+        return errors;
+    }
+    /**
+     * Validate required fields
+     */
+    validateRequiredFields(pack) {
+        const errors = [];
+        if (!pack.name || typeof pack.name !== 'string') {
+            errors.push({
+                code: 'QP2V003',
+                message: 'Pack name is required and must be a string',
+                path: 'name',
+                suggestion: 'Add: name: "my-pack"'
+            });
+        }
+        if (!pack.gates || typeof pack.gates !== 'object') {
+            errors.push({
+                code: 'QP2V004',
+                message: 'Gates configuration is required and must be an object',
+                path: 'gates',
+                suggestion: 'Add: gates: { smoke: { adapter: "playwright-api", test_files: ["tests/**/*.spec.ts"] } }'
+            });
+        }
+        if (pack.gates && Object.keys(pack.gates).length === 0) {
+            errors.push({
+                code: 'QP2V005',
+                message: 'At least one gate must be defined',
+                path: 'gates',
+                suggestion: 'Add at least one gate configuration'
+            });
+        }
+        return errors;
+    }
+    /**
+     * Validate auth configuration
+     */
+    async validateAuthConfig(auth) {
+        const errors = [];
+        const warnings = [];
+        if (!auth) {
+            return { errors, warnings };
+        }
+        // Validate default auth profiles reference existing profiles
+        const profiles = auth.profiles || {};
+        const validAuthTypes = ['none', 'jwt', 'oauth2', 'api_key', 'bearer', 'basic', 'totp', 'ui_login', 'gcp_adc', 'aws_iam', 'azure_ad'];
+        if (auth.api && !profiles[auth.api]) {
+            errors.push({
+                code: 'QP2V006',
+                message: `Auth profile "${auth.api}" referenced in auth.api but not defined in profiles`,
+                path: 'auth.api',
+                suggestion: `Define the profile or use an existing one: ${Object.keys(profiles).join(', ') || 'none defined'}`
+            });
+        }
+        if (auth.ui && !profiles[auth.ui]) {
+            errors.push({
+                code: 'QP2V007',
+                message: `Auth profile "${auth.ui}" referenced in auth.ui but not defined in profiles`,
+                path: 'auth.ui',
+                suggestion: `Define the profile or use an existing one: ${Object.keys(profiles).join(', ') || 'none defined'}`
+            });
+        }
+        // Validate each profile
+        for (const [name, profile] of Object.entries(profiles)) {
+            const profileErrors = this.validateAuthProfile(name, profile, validAuthTypes);
+            errors.push(...profileErrors.errors);
+            warnings.push(...profileErrors.warnings);
+        }
+        return { errors, warnings };
+    }
+    /**
+     * Validate a single auth profile
+     */
+    validateAuthProfile(name, profile, validTypes) {
+        const errors = [];
+        const warnings = [];
+        if (!validTypes.includes(profile.type)) {
+            errors.push({
+                code: 'QP2V008',
+                message: `Invalid auth type: ${profile.type}`,
+                path: `auth.profiles.${name}.type`,
+                suggestion: `Use one of: ${validTypes.join(', ')}`
+            });
+        }
+        // Type-specific validation
+        switch (profile.type) {
+            case 'jwt':
+                if (!profile.config?.client_id && !profile.config?.token_endpoint) {
+                    warnings.push({
+                        code: 'QP2V009',
+                        message: 'JWT profile should have client_id or token_endpoint',
+                        path: `auth.profiles.${name}.config`,
+                        suggestion: 'Add client_id or token_endpoint for token retrieval'
+                    });
+                }
+                break;
+            case 'oauth2':
+                if (!profile.config?.token_url) {
+                    errors.push({
+                        code: 'QP2V010',
+                        message: 'OAuth2 profile requires token_url',
+                        path: `auth.profiles.${name}.config.token_url`,
+                        suggestion: 'Add token_url: "https://auth.example.com/oauth2/token"'
+                    });
+                }
+                if (!profile.config?.client_id) {
+                    errors.push({
+                        code: 'QP2V011',
+                        message: 'OAuth2 profile requires client_id',
+                        path: `auth.profiles.${name}.config.client_id`,
+                        suggestion: 'Add client_id: "your-client-id"'
+                    });
+                }
+                break;
+            case 'api_key':
+                if (!profile.config?.key) {
+                    errors.push({
+                        code: 'QP2V012',
+                        message: 'API key profile requires key',
+                        path: `auth.profiles.${name}.config.key`,
+                        suggestion: 'Add key or use secret reference: ${{ secrets.API_KEY }}'
+                    });
+                }
+                break;
+            case 'bearer':
+                if (!profile.config?.token) {
+                    errors.push({
+                        code: 'QP2V013',
+                        message: 'Bearer token profile requires token',
+                        path: `auth.profiles.${name}.config.token`,
+                        suggestion: 'Add token or use secret reference: ${{ secrets.BEARER_TOKEN }}'
+                    });
+                }
+                break;
+            case 'basic':
+                if (!profile.config?.username || !profile.config?.password) {
+                    warnings.push({
+                        code: 'QP2V014',
+                        message: 'Basic auth profile should have username and password',
+                        path: `auth.profiles.${name}.config`,
+                        suggestion: 'Add username and password or use secret references'
+                    });
+                }
+                break;
+            case 'ui_login':
+                if (!profile.config?.url) {
+                    errors.push({
+                        code: 'QP2V015',
+                        message: 'UI login profile requires url',
+                        path: `auth.profiles.${name}.config.url`,
+                        suggestion: 'Add url: "https://example.com/login"'
+                    });
+                }
+                break;
+            case 'totp':
+                if (!profile.config?.secret) {
+                    errors.push({
+                        code: 'QP2V016',
+                        message: 'TOTP profile requires secret',
+                        path: `auth.profiles.${name}.config.secret`,
+                        suggestion: 'Add TOTP secret or use secret reference'
+                    });
+                }
+                break;
+        }
+        return { errors, warnings };
+    }
+    /**
+     * Validate gates configuration
+     */
+    async validateGates(gates, checkFilesExist) {
+        const errors = [];
+        const warnings = [];
+        const frameworks = new Set();
+        let totalTests = 0;
+        if (!gates) {
+            return { errors, warnings, totalTests, frameworks: [] };
+        }
+        const validAdapters = [
+            'playwright-api',
+            'playwright-ui',
+            'k6-perf',
+            'semgrep-sast',
+            'zap-dast',
+            'gitleaks-secrets',
+            'osv-deps'
+        ];
+        for (const [gateName, gate] of Object.entries(gates)) {
+            // Validate adapter
+            if (gate.adapter && !validAdapters.includes(gate.adapter)) {
+                warnings.push({
+                    code: 'QP2V017',
+                    message: `Unknown adapter: ${gate.adapter}`,
+                    path: `gates.${gateName}.adapter`,
+                    suggestion: `Use one of: ${validAdapters.join(', ')}`
+                });
+            }
+            // Collect adapter type
+            if (gate.adapter) {
+                frameworks.add(gate.adapter);
+            }
+            // Validate test_files
+            if (gate.test_files && gate.test_files.length > 0) {
+                for (const pattern of gate.test_files) {
+                    const patternErrors = this.validateTestPattern(gateName, pattern, checkFilesExist);
+                    errors.push(...patternErrors.errors);
+                    warnings.push(...patternErrors.warnings);
+                    // Count matching files
+                    if (checkFilesExist) {
+                        try {
+                            const matches = await glob(pattern, { cwd: this.baseDir, absolute: false });
+                            totalTests += matches.length;
+                        }
+                        catch {
+                            // Ignore glob errors
+                        }
+                    }
+                }
+            }
+            else if (!gate.config && !gate.test_files) {
+                warnings.push({
+                    code: 'QP2V018',
+                    message: `Gate "${gateName}" has neither test_files nor config`,
+                    path: `gates.${gateName}`,
+                    suggestion: 'Add test_files: ["tests/**/*.spec.ts"] or config with inline configuration'
+                });
+            }
+            // Validate auth profile reference
+            if (gate.auth) {
+                // Will be validated against profiles in auth section
+                if (!gate.adapter?.includes('api') && !gate.adapter?.includes('ui')) {
+                    warnings.push({
+                        code: 'QP2V019',
+                        message: `Auth profile specified but adapter may not support auth`,
+                        path: `gates.${gateName}.auth`,
+                        suggestion: 'Ensure adapter supports authentication'
+                    });
+                }
+            }
+            // Validate budgets
+            if (gate.budgets) {
+                const budgetErrors = this.validateGateBudgets(gateName, gate.budgets);
+                errors.push(...budgetErrors);
+            }
+            // Validate dependencies
+            if (gate.depends_on) {
+                for (const dep of gate.depends_on) {
+                    if (!gates[dep]) {
+                        errors.push({
+                            code: 'QP2V020',
+                            message: `Gate depends on non-existent gate: ${dep}`,
+                            path: `gates.${gateName}.depends_on`,
+                            suggestion: `Remove ${dep} from dependencies or define the gate`
+                        });
+                    }
+                }
+            }
+        }
+        return { errors, warnings, totalTests, frameworks: Array.from(frameworks) };
+    }
+    /**
+     * Validate a test file pattern
+     */
+    validateTestPattern(gateName, pattern, checkExistence) {
+        const errors = [];
+        const warnings = [];
+        // Check for invalid patterns
+        if (pattern.includes('..')) {
+            errors.push({
+                code: 'QP2V021',
+                message: 'Test pattern cannot contain parent directory reference ".."',
+                path: `gates.${gateName}.test_files`,
+                suggestion: 'Use relative patterns without ..'
+            });
+        }
+        if (pattern.startsWith('/')) {
+            warnings.push({
+                code: 'QP2V022',
+                message: 'Absolute path may not work across environments',
+                path: `gates.${gateName}.test_files`,
+                suggestion: 'Use relative pattern from pack file location'
+            });
+        }
+        return { errors, warnings };
+    }
+    /**
+     * Validate gate budgets
+     */
+    validateGateBudgets(gateName, budgets) {
+        const errors = [];
+        for (const [key, value] of Object.entries(budgets)) {
+            if (typeof value !== 'number' || value < 0) {
+                errors.push({
+                    code: 'QP2V023',
+                    message: `Budget value must be a positive number: ${key}`,
+                    path: `gates.${gateName}.budgets.${key}`,
+                    suggestion: 'Use a positive number value'
+                });
+            }
+        }
+        return errors;
+    }
+    /**
+     * Validate hooks configuration
+     */
+    validateHooks(hooks) {
+        const errors = [];
+        const warnings = [];
+        if (!hooks) {
+            return { errors, warnings };
+        }
+        const validHookTypes = ['run', 'wait_on', 'script', 'docker'];
+        const validPhases = ['beforeAll', 'afterAll', 'beforeEach', 'afterEach'];
+        for (const [phase, hookList] of Object.entries(hooks)) {
+            if (!validPhases.includes(phase)) {
+                warnings.push({
+                    code: 'QP2V024',
+                    message: `Unknown hook phase: ${phase}`,
+                    path: `hooks.${phase}`,
+                    suggestion: `Use one of: ${validPhases.join(', ')}`
+                });
+                continue;
+            }
+            if (!Array.isArray(hookList)) {
+                errors.push({
+                    code: 'QP2V025',
+                    message: `Hook phase must be an array: ${phase}`,
+                    path: `hooks.${phase}`,
+                    suggestion: 'Use array format: hooks: { beforeAll: [{ run: "command" }] }'
+                });
+                continue;
+            }
+            for (let i = 0; i < hookList.length; i++) {
+                const hook = hookList[i];
+                const hookErrors = this.validateHook(phase, i, hook, validHookTypes);
+                errors.push(...hookErrors.errors);
+                warnings.push(...hookErrors.warnings);
+            }
+        }
+        return { errors, warnings };
+    }
+    /**
+     * Validate a single hook
+     */
+    validateHook(phase, index, hook, validTypes) {
+        const errors = [];
+        const warnings = [];
+        if (!hook.type || !validTypes.includes(hook.type)) {
+            errors.push({
+                code: 'QP2V026',
+                message: `Invalid or missing hook type: ${hook.type}`,
+                path: `hooks.${phase}[${index}].type`,
+                suggestion: `Use one of: ${validTypes.join(', ')}`
+            });
+            return { errors, warnings };
+        }
+        // Type-specific validation
+        switch (hook.type) {
+            case 'run':
+            case 'script':
+                if (!hook.command) {
+                    errors.push({
+                        code: 'QP2V027',
+                        message: `${hook.type} hook requires command`,
+                        path: `hooks.${phase}[${index}].command`,
+                        suggestion: 'Add command to execute'
+                    });
+                }
+                break;
+            case 'wait_on':
+                if (!hook.wait_for?.resource) {
+                    errors.push({
+                        code: 'QP2V028',
+                        message: 'wait_on hook requires resource',
+                        path: `hooks.${phase}[${index}].wait_for.resource`,
+                        suggestion: 'Add resource: "http://localhost:3000" or "tcp://localhost:8080"'
+                    });
+                }
+                break;
+            case 'docker':
+                if (!hook.compose) {
+                    warnings.push({
+                        code: 'QP2V029',
+                        message: 'Docker hook should have compose configuration',
+                        path: `hooks.${phase}[${index}]`,
+                        suggestion: 'Add compose: { services: ["app"] }'
+                    });
+                }
+                break;
+        }
+        // Check for dangerous commands
+        if (hook.command) {
+            const dangerous = ['rm -rf', 'sudo rm', 'del /f', 'format', 'mkfs'];
+            if (dangerous.some(d => hook.command.toLowerCase().includes(d))) {
+                warnings.push({
+                    code: 'QP2V030',
+                    message: 'Potentially dangerous command in hook',
+                    path: `hooks.${phase}[${index}].command`,
+                    suggestion: 'Review command for safety'
+                });
+            }
+        }
+        return { errors, warnings };
+    }
+    /**
+     * Validate execution configuration
+     */
+    validateExecutionConfig(execution) {
+        const errors = [];
+        const warnings = [];
+        if (!execution) {
+            // Add default if missing
+            return { errors, warnings };
+        }
+        // Validate on_failure
+        if (execution.on_failure && !['stop', 'continue', 'proceed'].includes(execution.on_failure)) {
+            errors.push({
+                code: 'QP2V031',
+                message: `Invalid on_failure value: ${execution.on_failure}`,
+                path: 'execution.on_failure',
+                suggestion: 'Use one of: stop, continue, proceed'
+            });
+        }
+        // Validate numeric values
+        if (execution.default_timeout !== undefined && (typeof execution.default_timeout !== 'number' || execution.default_timeout <= 0)) {
+            errors.push({
+                code: 'QP2V032',
+                message: 'default_timeout must be a positive number',
+                path: 'execution.default_timeout',
+                suggestion: 'Use timeout in milliseconds (e.g., 30000 for 30s)'
+            });
+        }
+        if (execution.max_concurrency !== undefined && (typeof execution.max_concurrency !== 'number' || execution.max_concurrency <= 0)) {
+            errors.push({
+                code: 'QP2V033',
+                message: 'max_concurrency must be a positive number',
+                path: 'execution.max_concurrency',
+                suggestion: 'Use a value >= 1'
+            });
+        }
+        return { errors, warnings };
+    }
+    /**
+     * Validate business rules
+     */
+    validateBusinessRules(pack) {
+        const errors = [];
+        const warnings = [];
+        // Early return if no gates (already caught by required validation)
+        if (!pack.gates || Object.keys(pack.gates).length === 0) {
+            return { errors, warnings };
+        }
+        // Check for API gate with api auth but no auth profile
+        const apiGates = Object.entries(pack.gates).filter(([_, g]) => g.adapter?.includes('api'));
+        if (apiGates.length > 0 && !pack.auth?.api && !pack.auth?.profiles) {
+            warnings.push({
+                code: 'QP2V034',
+                message: 'API gates defined but no default auth profile configured',
+                path: 'auth.api',
+                suggestion: 'Add auth.api or define profiles for API authentication'
+            });
+        }
+        // Check for UI gate with ui auth but no auth profile
+        const uiGates = Object.entries(pack.gates).filter(([_, g]) => g.adapter?.includes('ui'));
+        if (uiGates.length > 0 && !pack.auth?.ui && !pack.auth?.profiles) {
+            warnings.push({
+                code: 'QP2V035',
+                message: 'UI gates defined but no default auth profile configured',
+                path: 'auth.ui',
+                suggestion: 'Add auth.ui or define profiles for UI authentication'
+            });
+        }
+        // Check for gates without adapter
+        const gatesWithoutAdapter = Object.entries(pack.gates).filter(([_, g]) => !g.adapter);
+        if (gatesWithoutAdapter.length > 0) {
+            const names = gatesWithoutAdapter.map(([name]) => name).join(', ');
+            warnings.push({
+                code: 'QP2V036',
+                message: `Gates without adapter: ${names}`,
+                path: 'gates',
+                suggestion: 'Add adapter to each gate for proper execution'
+            });
+        }
+        return { errors, warnings };
+    }
+}

package/dist/core/regression/detector.d.ts

@@ -0,0 +1,107 @@
+/**
+ * Regression Detector
+ *
+ * Detects regressions using statistical methods.
+ * Supports Z-score, T-test, Mann-Whitney U, and percentile-based detection.
+ */
+import type { RegressionDetection, RegressionConfig, TimeSeriesPoint, Baseline } from './types.js';
+/**
+ * Regression Detector class
+ */
+export declare class RegressionDetector {
+    private config;
+    private baselines;
+    constructor(config?: Partial<RegressionConfig>);
+    /**
+     * Detect regressions by comparing current values to baseline
+     */
+    detectRegressions(currentData: Record<string, number>, currentRunId: string, gate: string, context?: Record<string, unknown>): RegressionDetection[];
+    /**
+     * Detect regression for a single metric
+     */
+    private detectMetricRegression;
+    /**
+     * Perform statistical test
+     */
+    private performStatisticalTest;
+    /**
+     * Z-score test
+     */
+    private zScoreTest;
+    /**
+     * T-test (simplified for single value vs baseline)
+     */
+    private tTest;
+    /**
+     * Percentile test
+     */
+    private percentileTest;
+    /**
+     * Calculate percentile for a value in baseline
+     */
+    private calculatePercentile;
+    /**
+     * Normal CDF (cumulative distribution function)
+     */
+    private normalCDF;
+    /**
+     * Student's t CDF approximation
+     */
+    private studentTCDF;
+    /**
+     * Check if change constitutes a regression
+     */
+    private isRegression;
+    /**
+     * Get direction of change (worse/better/neutral)
+     */
+    private getDIRECTION;
+    /**
+     * Calculate regression severity
+     */
+    private calculateSeverity;
+    /**
+     * Generate suggestions for regression
+     */
+    private generateSuggestions;
+    /**
+     * Update baseline with new data
+     */
+    updateBaseline(metricName: string, data: TimeSeriesPoint[]): void;
+    /**
+     * Get baseline for a metric
+     */
+    getBaseline(metricName: string): Baseline | undefined;
+    /**
+     * Get all baselines
+     */
+    getAllBaselines(): Baseline[];
+    /**
+     * Get threshold for a metric
+     */
+    private getThresholdForMetric;
+    /**
+     * Check if metric matches pattern
+     */
+    private metricMatchesPattern;
+    /**
+     * Get regression type for metric
+     */
+    private getTypeForMetric;
+    /**
+     * Extract component name from metric and context
+     */
+    private extractComponent;
+    /**
+     * Check if metric should be ignored
+     */
+    private shouldIgnore;
+    /**
+     * Create default configuration
+     */
+    private createDefaultConfig;
+}
+/**
+ * Create a regression detector
+ */
+export declare function createRegressionDetector(config?: Partial<RegressionConfig>): RegressionDetector;