qa360 2.2.1 → 2.2.13

package/core/src/pack/__tests__/validator.test.ts

@@ -1,759 +0,0 @@
-/**
- * PackValidator Tests - 0% → 70%+
- * Tests JSON Schema validation and business rules
- */
-
-import { describe, it, expect, beforeEach } from 'vitest';
-import { PackValidator } from '../validator.js';
-import type { PackConfigV1 } from '../../types/pack-v1.js';
-
-describe('PackValidator', () => {
-  let validator: PackValidator;
-
-  beforeEach(() => {
-    validator = new PackValidator();
-  });
-
-  describe('Schema Validation', () => {
-    it('should validate a valid minimal pack', () => {
-      const pack: PackConfigV1 = {
-        version: 1,
-        name: 'minimal-pack',
-        gates: ['api_smoke']
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(true);
-      expect(result.errors).toHaveLength(0);
-    });
-
-    it('should validate a complete pack with all fields', () => {
-      const pack: PackConfigV1 = {
-        version: 1,
-        name: 'complete-pack',
-        description: 'A complete test pack',
-        gates: ['api_smoke', 'ui', 'perf', 'sast'],
-        targets: {
-          api: {
-            baseUrl: 'https://api.example.com',
-            smoke: ['GET /health -> 200']
-          },
-          web: {
-            baseUrl: 'https://example.com',
-            pages: ['https://example.com']
-          }
-        },
-        budgets: {
-          perf_p95_ms: 2000,
-          a11y_min: 90
-        },
-        security: {
-          sast: {
-            max_critical: 0,
-            max_high: 3
-          }
-        },
-        hooks: {
-          beforeAll: [{ run: 'docker compose up -d', timeout: 30000 }],
-          afterAll: [{ run: 'docker compose down', timeout: 10000 }]
-        }
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(true);
-      expect(result.errors).toHaveLength(0);
-    });
-
-    it('should reject pack missing required field "version"', () => {
-      const pack: any = {
-        name: 'test-pack',
-        gates: ['api_smoke']
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-      expect(result.errors).toHaveLength(1);
-      expect(result.errors[0].code).toBe('QP100'); // required error
-      expect(result.errors[0].message).toContain('version');
-    });
-
-    it('should reject pack missing required field "name"', () => {
-      const pack: any = {
-        version: 1,
-        gates: ['api_smoke']
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-      expect(result.errors[0].message).toContain('name');
-    });
-
-    it('should reject pack missing required field "gates"', () => {
-      const pack: any = {
-        version: 1,
-        name: 'test-pack'
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-      expect(result.errors[0].message).toContain('gates');
-    });
-
-    it('should reject pack with invalid version number', () => {
-      const pack: any = {
-        version: 2,
-        name: 'test-pack',
-        gates: ['api_smoke']
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-      expect(result.errors[0].code).toBe('QP200'); // v2 not supported (version detection runs before schema validation)
-    });
-
-    it('should reject pack with invalid gate name', () => {
-      const pack: any = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['invalid_gate']
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-      expect(result.errors[0].code).toBe('QP104'); // enum error
-      expect(result.errors[0].message).toContain('api_smoke');
-    });
-
-    it('should reject pack with empty gates array', () => {
-      const pack: any = {
-        version: 1,
-        name: 'test-pack',
-        gates: []
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-      expect(result.errors[0].code).toBe('QP110'); // minItems error
-    });
-
-    it('should reject pack name with invalid characters', () => {
-      const pack: any = {
-        version: 1,
-        name: 'test pack!',
-        gates: ['api_smoke']
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-      expect(result.errors[0].code).toBe('QP103'); // pattern error
-      expect(result.errors[0].suggestion).toContain('letters, numbers, underscore');
-    });
-
-    it('should reject pack with invalid baseUrl format', () => {
-      const pack: any = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['api_smoke'],
-        targets: {
-          api: {
-            baseUrl: 'not-a-url'
-          }
-        }
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-      expect(result.errors[0].code).toBe('QP102'); // format error
-    });
-
-    it('should reject pack with invalid smoke test format', () => {
-      const pack: any = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['api_smoke'],
-        targets: {
-          api: {
-            baseUrl: 'https://api.example.com',
-            smoke: ['GET /health'] // missing -> 200
-          }
-        }
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-      expect(result.errors[0].code).toBe('QP103'); // pattern error
-      expect(result.errors[0].suggestion).toContain('GET /path -> 200');
-    });
-  });
-
-  describe('Business Rules Validation', () => {
-    it('should error when api_smoke gate has no api target (multi-gate pack)', () => {
-      const pack: PackConfigV1 = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['api_smoke', 'ui']
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-      expect(result.errors).toHaveLength(2); // api_smoke and ui
-      expect(result.errors[0].code).toBe('QP001');
-      expect(result.errors[0].message).toContain('API smoke gate requires api target');
-    });
-
-    it('should warn when api_smoke gate has no api target (minimal pack)', () => {
-      const pack: PackConfigV1 = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['api_smoke']
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(true);
-      expect(result.warnings).toHaveLength(1);
-      expect(result.warnings[0].code).toBe('QP001');
-      expect(result.warnings[0].message).toContain('default configuration');
-    });
-
-    it('should error when ui gate has no web target', () => {
-      const pack: PackConfigV1 = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['ui']
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-      expect(result.errors[0].code).toBe('QP002');
-      expect(result.errors[0].message).toContain('UI gate requires web target');
-    });
-
-    it('should warn when perf gate has no budget', () => {
-      const pack: PackConfigV1 = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['perf']
-      };
-      const result = validator.validate(pack);
-      expect(result.warnings).toHaveLength(1);
-      expect(result.warnings[0].code).toBe('QP003');
-      expect(result.warnings[0].message).toContain('Performance gate without budget');
-    });
-
-    it('should warn when a11y gate has no budget', () => {
-      const pack: PackConfigV1 = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['a11y']
-      };
-      const result = validator.validate(pack);
-      expect(result.warnings).toHaveLength(1);
-      expect(result.warnings[0].code).toBe('QP004');
-      expect(result.warnings[0].message).toContain('Accessibility gate without budget');
-    });
-
-    it('should warn when security gates have no security config', () => {
-      const pack: PackConfigV1 = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['sast', 'dast']
-      };
-      const result = validator.validate(pack);
-      expect(result.warnings).toHaveLength(1);
-      expect(result.warnings[0].code).toBe('QP005');
-      expect(result.warnings[0].message).toContain('Security gates without security configuration');
-    });
-
-    it('should detect potential secrets in environment', () => {
-      const pack: PackConfigV1 = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['api_smoke'],
-        environment: {
-          API_KEY: 'sk-1234567890abcdefghijklmnopqrstuvwxyz'
-        }
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-      expect(result.errors[0].code).toBe('QP006');
-      expect(result.errors[0].message).toContain('Potential secret detected');
-      expect(result.errors[0].suggestion).toContain('${{ secrets.API_KEY }}');
-    });
-
-    it('should allow secret references in environment', () => {
-      const pack: PackConfigV1 = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['api_smoke'],
-        environment: {
-          API_KEY: '${{ secrets.API_KEY }}'
-        }
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(true);
-      expect(result.errors).toHaveLength(0);
-    });
-
-    it('should warn about dangerous hook commands', () => {
-      const pack: PackConfigV1 = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['api_smoke'],
-        targets: {
-          api: {
-            baseUrl: 'https://api.example.com'
-          }
-        },
-        hooks: {
-          beforeAll: [
-            { run: 'rm -rf /tmp/test' }
-          ]
-        }
-      };
-      const result = validator.validate(pack);
-      expect(result.warnings.length).toBeGreaterThanOrEqual(1);
-      const dangWarning = result.warnings.find(w => w.code === 'QP007');
-      expect(dangWarning).toBeDefined();
-      expect(dangWarning?.message).toContain('dangerous command');
-    });
-
-    it('should warn about sudo rm commands in hooks', () => {
-      const pack: PackConfigV1 = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['api_smoke'],
-        targets: {
-          api: {
-            baseUrl: 'https://api.example.com'
-          }
-        },
-        hooks: {
-          beforeAll: [
-            { run: 'sudo rm -rf /tmp/test' }
-          ]
-        }
-      };
-      const result = validator.validate(pack);
-      expect(result.warnings.length).toBeGreaterThanOrEqual(1);
-      const sudoWarning = result.warnings.find(w => w.code === 'QP007');
-      expect(sudoWarning).toBeDefined();
-    });
-
-    it('should detect multiple dangerous patterns', () => {
-      const pack: PackConfigV1 = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['api_smoke'],
-        hooks: {
-          beforeAll: [
-            { run: 'rm -rf /tmp' },
-            { run: 'sudo rm -rf /var/log' }
-          ]
-        }
-      };
-      const result = validator.validate(pack);
-      expect(result.warnings.length).toBeGreaterThanOrEqual(2);
-    });
-  });
-
-  describe('Error Formatting', () => {
-    it('should provide helpful suggestion for missing version', () => {
-      const pack: any = {
-        name: 'test-pack',
-        gates: ['api_smoke']
-      };
-      const result = validator.validate(pack);
-      expect(result.errors[0].suggestion).toBe('Add: version: 1');
-    });
-
-    it('should provide helpful suggestion for missing name', () => {
-      const pack: any = {
-        version: 1,
-        gates: ['api_smoke']
-      };
-      const result = validator.validate(pack);
-      expect(result.errors[0].suggestion).toBe('Add: name: "my-pack"');
-    });
-
-    it('should provide helpful suggestion for missing gates', () => {
-      const pack: any = {
-        version: 1,
-        name: 'test-pack'
-      };
-      const result = validator.validate(pack);
-      expect(result.errors[0].suggestion).toBe('Add: gates: ["api_smoke"]');
-    });
-
-    it('should provide enum suggestions', () => {
-      const pack: any = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['invalid_gate']
-      };
-      const result = validator.validate(pack);
-      expect(result.errors[0].suggestion).toContain('api_smoke');
-    });
-
-    it('should provide URI format suggestion', () => {
-      const pack: any = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['api_smoke'],
-        targets: {
-          api: {
-            baseUrl: 'example.com'
-          }
-        }
-      };
-      const result = validator.validate(pack);
-      const formatError = result.errors.find(e => e.code === 'QP102');
-      expect(formatError?.suggestion).toContain('https://');
-    });
-  });
-
-  describe('Complex Scenarios', () => {
-    it('should handle pack with multiple errors and warnings', () => {
-      const pack: any = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['api_smoke', 'ui', 'perf'],
-        environment: {
-          SECRET: 'sk-abcdefghijklmnopqrstuvwxyz1234567890'
-        },
-        hooks: {
-          beforeAll: [{ run: 'rm -rf /tmp' }]
-        }
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-      expect(result.errors.length).toBeGreaterThan(0); // QP001, QP002, QP006
-      expect(result.warnings.length).toBeGreaterThan(0); // QP003, QP007
-    });
-
-    it('should validate pack with all gate types', () => {
-      const pack: PackConfigV1 = {
-        version: 1,
-        name: 'all-gates-pack',
-        gates: ['api_smoke', 'ui', 'perf', 'sast', 'dast', 'a11y'],
-        targets: {
-          api: {
-            baseUrl: 'https://api.example.com',
-            smoke: ['GET /health -> 200']
-          },
-          web: {
-            baseUrl: 'https://example.com',
-            pages: ['https://example.com']
-          }
-        },
-        budgets: {
-          perf_p95_ms: 2000,
-          a11y_min: 90
-        },
-        security: {
-          sast: {
-            max_critical: 0,
-            max_high: 3
-          },
-          dast: {
-            max_high: 5
-          }
-        }
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(true);
-      expect(result.errors).toHaveLength(0);
-    });
-
-    it('should handle pack with run hooks', () => {
-      const pack: PackConfigV1 = {
-        version: 1,
-        name: 'run-hooks-pack',
-        gates: ['api_smoke'],
-        targets: {
-          api: {
-            baseUrl: 'https://api.example.com'
-          }
-        },
-        hooks: {
-          beforeAll: [
-            { run: 'docker compose up -d', timeout: 30000 },
-            { run: 'sleep 5', timeout: 10000 }
-          ],
-          afterAll: [
-            { run: 'docker compose down' }
-          ]
-        }
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(true);
-    });
-
-    it('should detect GitHub token pattern', () => {
-      const pack: PackConfigV1 = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['api_smoke'],
-        environment: {
-          GH_TOKEN: 'ghp_' + 'a'.repeat(36)
-        }
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-      expect(result.errors[0].code).toBe('QP006');
-    });
-
-    it('should detect base64-like secret', () => {
-      const pack: PackConfigV1 = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['api_smoke'],
-        environment: {
-          TOKEN: 'dGhpc2lzYXRva2VuMTIzNDU2Nzg5MA=='
-        }
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-      expect(result.errors[0].code).toBe('QP006');
-    });
-
-    it('should allow normal environment variables', () => {
-      const pack: PackConfigV1 = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['api_smoke'],
-        environment: {
-          NODE_ENV: 'production',
-          PORT: '3000',
-          DEBUG: 'true'
-        }
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(true);
-    });
-  });
-
-  describe('Edge Cases and Additional Coverage', () => {
-    it('should handle unknown error keywords gracefully', () => {
-      const pack: any = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['api_smoke', 'api_smoke'] // Duplicate items to trigger uniqueItems error
-      };
-      const result = validator.validate(pack);
-      // uniqueItems error should be formatted with default message
-      expect(result.valid).toBe(false);
-      expect(result.errors.length).toBeGreaterThan(0);
-    });
-
-    it('should handle additional properties error', () => {
-      const pack: any = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['api_smoke'],
-        unknownField: 'should not be here'
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-      const error = result.errors.find(e => e.message.includes('Unknown property'));
-      expect(error).toBeDefined();
-    });
-
-    it('should handle format errors for non-URI formats', () => {
-      const pack: any = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['api_smoke'],
-        targets: {
-          api: {
-            baseUrl: 'not-a-uri',
-            smoke: ['invalid format'] // Pattern error
-          }
-        }
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-      // Should have both format and pattern errors
-      expect(result.errors.length).toBeGreaterThan(0);
-    });
-
-    it('should provide suggestion for pattern error in name field', () => {
-      const pack: any = {
-        version: 1,
-        name: 'test pack!',
-        gates: ['api_smoke']
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-      const patternError = result.errors.find(e => e.code === 'QP103');
-      expect(patternError?.suggestion).toContain('letters, numbers, underscore');
-    });
-
-    it('should handle error without message field', () => {
-      const pack: any = {
-        version: 1,
-        name: 'test-pack',
-        gates: [] // Empty array to trigger minItems
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-      expect(result.errors.length).toBeGreaterThan(0);
-    });
-
-    it('should handle unknown format type', () => {
-      // Create a scenario that triggers an unknown format error
-      // This is difficult with the current schema, so we test the default path
-      const pack: any = {
-        version: 2, // v2 not yet supported - triggers QP200
-        name: 'test-pack',
-        gates: ['api_smoke']
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-      expect(result.errors[0].code).toBe('QP200');
-    });
-
-    it('should return valid result with warnings only', () => {
-      const pack: PackConfigV1 = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['perf'] // No budget, should trigger warning but valid
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(true);
-      expect(result.warnings.length).toBeGreaterThan(0);
-      expect(result.errors).toHaveLength(0);
-    });
-
-    it('should handle minimal api_smoke pack correctly', () => {
-      const pack: PackConfigV1 = {
-        version: 1,
-        name: 'minimal-smoke',
-        gates: ['api_smoke']
-        // No targets - should trigger warning for minimal pack
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(true);
-      expect(result.warnings).toHaveLength(1);
-      expect(result.warnings[0].code).toBe('QP001');
-      expect(result.warnings[0].message).toContain('default configuration');
-    });
-
-    it('should handle error object without message property', () => {
-      // Test with a pack that triggers an error type that might not have message
-      const pack: any = {
-        version: '1', // Wrong type - should be number
-        name: 'test-pack',
-        gates: ['api_smoke']
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-      expect(result.errors.length).toBeGreaterThan(0);
-    });
-
-    it('should handle pattern errors in non-name fields', () => {
-      const pack: any = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['api_smoke'],
-        targets: {
-          api: {
-            baseUrl: 'https://example.com',
-            smoke: ['GET /path'] // Missing -> 200 part
-          }
-        }
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-      const patternError = result.errors.find(e => e.code === 'QP103');
-      expect(patternError).toBeDefined();
-    });
-
-    it('should handle type errors for version field', () => {
-      const pack: any = {
-        version: 'not-a-number',
-        name: 'test-pack',
-        gates: ['api_smoke']
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-      expect(result.errors.length).toBeGreaterThan(0);
-    });
-
-    it('should handle errors with undefined message', () => {
-      const pack: any = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['api_smoke'],
-        targets: {
-          api: {
-            baseUrl: 123 // Wrong type for baseUrl
-          }
-        }
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-    });
-
-    it('should handle format errors with unknown format type', () => {
-      const pack: any = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['api_smoke'],
-        targets: {
-          api: {
-            baseUrl: 123 // Type error, not format error, but helps coverage
-          }
-        }
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-    });
-
-    it('should handle pattern errors in non-name, non-smoke fields', () => {
-      const pack: any = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['api_smoke'],
-        environment: {
-          'INVALID KEY!': 'value' // If environment keys have patterns
-        }
-      };
-      const result = validator.validate(pack);
-      // This may or may not fail depending on schema, but covers the else path
-      expect(result).toBeDefined();
-    });
-
-    it('should handle maxLength errors', () => {
-      const pack: any = {
-        version: 1,
-        name: 'a'.repeat(300), // Very long name to trigger maxLength if it exists
-        gates: ['api_smoke']
-      };
-      const result = validator.validate(pack);
-      // Might fail or pass depending on schema, helps cover error branches
-      expect(result).toBeDefined();
-    });
-
-    it('should handle minimum/maximum errors', () => {
-      const pack: any = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['api_smoke'],
-        budgets: {
-          perf_p95_ms: -1 // Negative value if min is 0
-        }
-      };
-      const result = validator.validate(pack);
-      expect(result).toBeDefined();
-    });
-
-    it('should validate pack with empty targets object', () => {
-      const pack: PackConfigV1 = {
-        version: 1,
-        name: 'test-pack',
-        gates: ['api_smoke'],
-        targets: {} // Empty targets object
-      };
-      const result = validator.validate(pack);
-      // With only api_smoke gate, it's a warning not an error (minimal pack)
-      expect(result.valid).toBe(true);
-      expect(result.warnings).toHaveLength(1);
-    });
-
-    it('should handle type errors with number instead of string', () => {
-      const pack: any = {
-        version: 1,
-        name: 123, // Number instead of string
-        gates: ['api_smoke']
-      };
-      const result = validator.validate(pack);
-      expect(result.valid).toBe(false);
-    });
-  });
-});