yamltest 1.0.4 → 1.1.0

package/README.md

@@ -82,10 +82,31 @@ Exit codes: `0` = all passed, `1` = one or more failed.
 
 ---
 
+## Input validation
+
+All test definitions are validated against a strict schema **before** any test executes. Invalid input produces clear error messages pointing to the exact location of the problem:
+
+```
+Validation failed:
+
+  Test #1 ("login") /http/method: must be one of: GET, POST, PUT, DELETE, PATCH
+  Test #3: must define exactly one of: http, command, wait, httpBodyComparison
+  Test #5 ("check pods") /source: missing required property "selector"
+```
+
+Validation checks include:
+- Exactly one test type per definition (`http`, `command`, `wait`, or `httpBodyComparison`)
+- Required fields per test type (e.g., `command.command`, `wait.target`)
+- Value types and allowed enums (e.g., `source.type` must be `local` or `pod`)
+- Conditional requirements (e.g., `source.type: pod` requires `selector`; `setVars` requires `expect` for HTTP/command tests)
+- Unknown property detection on sub-objects (catches typos like `htpp` instead of `http`)
+
+---
+
 ## Programmatic API
 
 ```js
-const { runTests, executeTest } = require('yamltest');
+const { runTests, executeTest, validateTestDefinitions } = require('yamltest');
 
 // Run one or more tests from a YAML string (array or single object)
 const result = await runTests(yamlString);
@@ -94,6 +115,13 @@ console.log(result.passed, result.failed, result.skipped, result.total);
 
 // Run a single test (low-level)
 await executeTest(yamlString); // returns true or throws
+
+// Validate test definitions without executing them
+const yaml = require('js-yaml');
+const definitions = Array.isArray(yaml.load(yamlString))
+  ? yaml.load(yamlString)
+  : [yaml.load(yamlString)];
+validateTestDefinitions(definitions); // throws on validation errors
 ```
 
 ---
@@ -550,7 +578,8 @@ Prints full request/response details, comparison results, and kubectl commands.
 ```
 src/
   core.js       # Test execution engine (HTTP, command, wait, comparison)
-  runner.js     # Multi-test orchestration (YAML parsing, fail-fast, retry)
+  runner.js     # Multi-test orchestration (YAML parsing, validation, fail-fast, retry)
+  validate.js   # JSON Schema validation (Ajv)
   index.js      # Public API
   cli.js        # YAMLTest binary entry point
 test/

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "yamltest",
-  "version": "1.0.4",
+  "version": "1.1.0",
   "description": "Declarative YAML-based HTTP, command, and Kubernetes test runner",
   "main": "./src/index.js",
   "bin": {
@@ -34,6 +34,8 @@
   "author": "",
   "license": "MIT",
   "dependencies": {
+    "ajv": "^8.18.0",
+    "ajv-errors": "^3.0.0",
     "axios": "^1.6.0",
     "deep-diff": "^1.0.2",
     "fs-extra": "^11.2.0",

package/src/core.js

@@ -81,6 +81,10 @@ async function executeTest(yamlDefinition) {
     throw new Error('Invalid test definition: expected a YAML object');
   }
 
+  // Validate against schema before execution
+  const { validateTestDefinitions } = require('./validate');
+  validateTestDefinitions([testConfig]);
+
   // Dispatch to the appropriate test function based on what's defined
   if (testConfig.http) {
     debugLog('Detected HTTP test, dispatching to executeHttpTest');
@@ -612,7 +616,7 @@ async function executeHttpTest(test) {
       test.http.url = `${parsedUrl.protocol}//${parsedUrl.host}`;
       // Prepend the extracted path to any explicit path (avoid double slashes)
       const explicitPath = test.http.path || '';
-      test.http.path = urlPath.replace(/\/$/, '') + (explicitPath.startsWith('/') ? explicitPath : '/' + explicitPath);
+      test.http.path = urlPath.replace(/\/$/, '') + (explicitPath ? (explicitPath.startsWith('/') ? explicitPath : '/' + explicitPath) : '');
     }
   }
 

package/src/index.js

@@ -18,6 +18,9 @@
 // Multi-test orchestration layer
 const { runTests, parseTestDefinitions, runSingleTest } = require('./runner');
 
+// Schema validation
+const { validateTestDefinitions } = require('./validate');
+
 // Low-level core exports (single-test API)
 const {
   executeTest,
@@ -35,6 +38,7 @@ module.exports = {
   runTests,
   parseTestDefinitions,
   runSingleTest,
+  validateTestDefinitions,
 
   // Low-level single-test API (v2.js re-exports)
   executeTest,

package/src/runner.js

@@ -2,6 +2,7 @@
 
 const yaml = require('js-yaml');
 const { executeTest } = require('./core');
+const { validateTestDefinitions } = require('./validate');
 
 /**
  * Parse a YAML string into an array of test definitions.
@@ -123,6 +124,10 @@ async function runSingleTest(def, index) {
  */
 async function runTests(yamlString) {
   const definitions = parseTestDefinitions(yamlString);
+
+  // Validate all definitions before executing any test
+  validateTestDefinitions(definitions);
+
   const total = definitions.length;
   const results = [];
 

package/src/validate.js

@@ -0,0 +1,628 @@
+'use strict';
+
+const Ajv = require('ajv');
+const ajvErrors = require('ajv-errors');
+
+// ── Reusable sub-schemas ─────────────────────────────────────────────
+
+const comparatorEnum = {
+  type: 'string',
+  enum: ['equals', 'contains', 'matches', 'exists', 'greaterThan', 'lessThan'],
+};
+
+const jsonPathExpectationItem = {
+  type: 'object',
+  required: ['path', 'comparator'],
+  properties: {
+    path: { type: 'string' },
+    comparator: comparatorEnum,
+    value: {},               // any type
+    negate: { type: 'boolean' },
+  },
+  additionalProperties: false,
+};
+
+const headerExpectationItem = {
+  type: 'object',
+  required: ['name', 'comparator'],
+  properties: {
+    name: { type: 'string' },
+    comparator: { type: 'string', enum: ['equals', 'contains', 'matches', 'exists'] },
+    value: { type: 'string' },
+    negate: { type: 'boolean' },
+  },
+  additionalProperties: false,
+};
+
+// bodyContains / bodyRegex accept: string | object | array of (string|object)
+const bodyContainsItem = {
+  oneOf: [
+    { type: 'string' },
+    {
+      type: 'object',
+      required: ['value'],
+      properties: {
+        value: { type: 'string' },
+        negate: { type: 'boolean' },
+        matchword: { type: 'boolean' },
+      },
+      additionalProperties: false,
+    },
+  ],
+};
+
+const bodyContainsSchema = {
+  oneOf: [
+    { type: 'string' },
+    {
+      type: 'object',
+      required: ['value'],
+      properties: {
+        value: { type: 'string' },
+        negate: { type: 'boolean' },
+        matchword: { type: 'boolean' },
+      },
+      additionalProperties: false,
+    },
+    {
+      type: 'array',
+      items: bodyContainsItem,
+      minItems: 1,
+    },
+  ],
+};
+
+const bodyRegexItem = {
+  oneOf: [
+    { type: 'string' },
+    {
+      type: 'object',
+      required: ['value'],
+      properties: {
+        value: { type: 'string' },
+        negate: { type: 'boolean' },
+      },
+      additionalProperties: false,
+    },
+  ],
+};
+
+const bodyRegexSchema = {
+  oneOf: [
+    { type: 'string' },
+    {
+      type: 'object',
+      required: ['value'],
+      properties: {
+        value: { type: 'string' },
+        negate: { type: 'boolean' },
+      },
+      additionalProperties: false,
+    },
+    {
+      type: 'array',
+      items: bodyRegexItem,
+      minItems: 1,
+    },
+  ],
+};
+
+// Output expectation for command stdout/stderr/output
+const outputExpectationItem = {
+  oneOf: [
+    { type: 'string' },
+    {
+      type: 'object',
+      properties: {
+        contains: { type: 'string' },
+        equals: { type: 'string' },
+        matches: { type: 'string' },
+        regex: { type: 'string' },
+        exists: { type: 'boolean' },
+        negate: { type: 'boolean' },
+        comparator: comparatorEnum,
+        value: {},
+      },
+      additionalProperties: false,
+    },
+    {
+      type: 'array',
+      items: {
+        oneOf: [
+          { type: 'string' },
+          {
+            type: 'object',
+            properties: {
+              contains: { type: 'string' },
+              equals: { type: 'string' },
+              matches: { type: 'string' },
+              regex: { type: 'string' },
+              exists: { type: 'boolean' },
+              negate: { type: 'boolean' },
+              comparator: comparatorEnum,
+              value: {},
+            },
+            additionalProperties: false,
+          },
+        ],
+      },
+      minItems: 1,
+    },
+  ],
+};
+
+// ── K8s selector ─────────────────────────────────────────────────────
+
+const k8sSelector = {
+  type: 'object',
+  required: ['kind', 'metadata'],
+  properties: {
+    kind: { type: 'string' },
+    context: { type: 'string' },
+    metadata: {
+      type: 'object',
+      properties: {
+        namespace: { type: 'string' },
+        name: { type: 'string' },
+        labels: {
+          type: 'object',
+          additionalProperties: { type: 'string' },
+        },
+      },
+      anyOf: [
+        { required: ['name'] },
+        { required: ['labels'] },
+      ],
+      additionalProperties: false,
+    },
+  },
+  additionalProperties: false,
+};
+
+// ── Source ────────────────────────────────────────────────────────────
+
+const sourceSchema = {
+  type: 'object',
+  required: ['type'],
+  properties: {
+    type: { type: 'string', enum: ['local', 'pod'] },
+    selector: k8sSelector,
+    container: { type: 'string' },
+    usePortForward: { type: 'boolean' },
+    usePodExec: { type: 'boolean' },
+  },
+  if: { properties: { type: { const: 'pod' } } },
+  then: { required: ['selector'] },
+  additionalProperties: false,
+};
+
+// ── HTTP config ──────────────────────────────────────────────────────
+
+const httpConfigSchema = {
+  type: 'object',
+  properties: {
+    url: { type: 'string' },
+    method: { type: 'string', enum: ['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'get', 'post', 'put', 'delete', 'patch'] },
+    path: { type: 'string' },
+    headers: {
+      type: 'object',
+      additionalProperties: { type: 'string' },
+    },
+    params: {
+      type: 'object',
+      additionalProperties: {},
+    },
+    body: {},   // string or object
+    skipSslVerification: { type: 'boolean' },
+    maxRedirects: { type: 'number' },
+    cert: { type: 'string' },
+    key: { type: 'string' },
+    ca: { type: 'string' },
+    scheme: { type: 'string', enum: ['http', 'https'] },
+    port: { oneOf: [{ type: 'number' }, { type: 'string' }] },
+  },
+  additionalProperties: false,
+};
+
+// ── HTTP expect ──────────────────────────────────────────────────────
+
+const httpExpectSchema = {
+  type: 'object',
+  properties: {
+    statusCode: {
+      oneOf: [
+        { type: 'number' },
+        { type: 'array', items: { type: 'number' }, minItems: 1 },
+      ],
+    },
+    body: {},   // any type for exact match
+    bodyContains: bodyContainsSchema,
+    bodyRegex: bodyRegexSchema,
+    bodyJsonPath: {
+      type: 'array',
+      items: jsonPathExpectationItem,
+      minItems: 1,
+    },
+    headers: {
+      type: 'array',
+      items: headerExpectationItem,
+      minItems: 1,
+    },
+  },
+  additionalProperties: false,
+};
+
+// ── HTTP setVars ─────────────────────────────────────────────────────
+
+const httpSetVarRule = {
+  type: 'object',
+  properties: {
+    jsonPath: { type: 'string' },
+    header: { type: 'string' },
+    statusCode: { const: true },
+    body: { const: true },
+    regex: {
+      type: 'object',
+      required: ['pattern'],
+      properties: {
+        pattern: { type: 'string' },
+        group: { type: 'number' },
+      },
+      additionalProperties: false,
+    },
+  },
+  additionalProperties: false,
+};
+
+const httpSetVarsSchema = {
+  type: 'object',
+  additionalProperties: httpSetVarRule,
+};
+
+// ── Command config ───────────────────────────────────────────────────
+
+const commandConfigSchema = {
+  type: 'object',
+  required: ['command'],
+  properties: {
+    command: { type: 'string' },
+    parseJson: { type: 'boolean' },
+    env: {
+      type: 'object',
+      additionalProperties: { type: 'string' },
+    },
+    workingDir: { type: 'string' },
+  },
+  additionalProperties: false,
+};
+
+// ── Command expect ───────────────────────────────────────────────────
+
+const commandExpectSchema = {
+  type: 'object',
+  properties: {
+    exitCode: { type: 'number' },
+    stdout: outputExpectationItem,
+    stderr: outputExpectationItem,
+    output: outputExpectationItem,
+    json: {},   // any type for direct comparison
+    jsonPath: {
+      type: 'array',
+      items: jsonPathExpectationItem,
+      minItems: 1,
+    },
+  },
+  additionalProperties: false,
+};
+
+// ── Command setVars ──────────────────────────────────────────────────
+
+const commandSetVarRule = {
+  type: 'object',
+  properties: {
+    jsonPath: { type: 'string' },
+    stdout: { const: true },
+    stderr: { const: true },
+    exitCode: { const: true },
+    regex: {
+      type: 'object',
+      required: ['pattern'],
+      properties: {
+        pattern: { type: 'string' },
+        group: { type: 'number' },
+        source: { type: 'string', enum: ['stdout', 'stderr'] },
+      },
+      additionalProperties: false,
+    },
+  },
+  additionalProperties: false,
+};
+
+const commandSetVarsSchema = {
+  type: 'object',
+  additionalProperties: commandSetVarRule,
+};
+
+// ── Wait config ──────────────────────────────────────────────────────
+
+const waitConfigSchema = {
+  type: 'object',
+  required: ['target'],
+  properties: {
+    target: k8sSelector,
+    jsonPath: { type: 'string' },
+    jsonPathExpectation: {
+      type: 'object',
+      required: ['comparator'],
+      properties: {
+        comparator: comparatorEnum,
+        value: {},
+        negate: { type: 'boolean' },
+      },
+      additionalProperties: false,
+    },
+    polling: {
+      type: 'object',
+      properties: {
+        timeoutSeconds: { type: 'number' },
+        intervalSeconds: { type: 'number' },
+        maxRetries: { type: 'number' },
+      },
+      additionalProperties: false,
+    },
+  },
+  additionalProperties: false,
+};
+
+// ── Wait setVars ─────────────────────────────────────────────────────
+
+const waitSetVarRule = {
+  type: 'object',
+  properties: {
+    value: { const: true },
+  },
+  additionalProperties: false,
+};
+
+const waitSetVarsSchema = {
+  type: 'object',
+  additionalProperties: waitSetVarRule,
+};
+
+// ── HTTP body comparison config ──────────────────────────────────────
+
+const httpBodyComparisonRequestSchema = {
+  type: 'object',
+  required: ['http', 'source'],
+  properties: {
+    http: httpConfigSchema,
+    source: sourceSchema,
+  },
+  additionalProperties: false,
+};
+
+const httpBodyComparisonConfigSchema = {
+  type: 'object',
+  required: ['request1', 'request2'],
+  properties: {
+    request1: httpBodyComparisonRequestSchema,
+    request2: httpBodyComparisonRequestSchema,
+    parseAsJson: { type: 'boolean' },
+    delaySeconds: { type: 'number' },
+    removeJsonPaths: {
+      type: 'array',
+      items: { type: 'string' },
+    },
+  },
+  additionalProperties: false,
+};
+
+// ── Main test definition schema ──────────────────────────────────────
+
+const testDefinitionSchema = {
+  type: 'object',
+  required: ['source'],
+  properties: {
+    name: { type: 'string' },
+    retries: { type: 'integer', minimum: 0 },
+    source: sourceSchema,
+    http: httpConfigSchema,
+    command: commandConfigSchema,
+    wait: waitConfigSchema,
+    httpBodyComparison: httpBodyComparisonConfigSchema,
+    expect: {},     // validated conditionally per test type
+    setVars: {},    // validated conditionally per test type
+  },
+  // Exactly one test type must be present
+  oneOf: [
+    {
+      required: ['http'],
+      not: { anyOf: [{ required: ['command'] }, { required: ['wait'] }, { required: ['httpBodyComparison'] }] },
+    },
+    {
+      required: ['command'],
+      not: { anyOf: [{ required: ['http'] }, { required: ['wait'] }, { required: ['httpBodyComparison'] }] },
+    },
+    {
+      required: ['wait'],
+      not: { anyOf: [{ required: ['http'] }, { required: ['command'] }, { required: ['httpBodyComparison'] }] },
+    },
+    {
+      required: ['httpBodyComparison'],
+      not: { anyOf: [{ required: ['http'] }, { required: ['command'] }, { required: ['wait'] }] },
+    },
+  ],
+  // Conditional validation of expect and setVars per test type
+  allOf: [
+    // HTTP: validate expect and setVars shapes
+    {
+      if: { required: ['http'] },
+      then: {
+        properties: {
+          expect: httpExpectSchema,
+          setVars: httpSetVarsSchema,
+        },
+      },
+    },
+    // Command: validate expect and setVars shapes
+    {
+      if: { required: ['command'] },
+      then: {
+        properties: {
+          expect: commandExpectSchema,
+          setVars: commandSetVarsSchema,
+        },
+      },
+    },
+    // Wait: no expect, setVars uses wait-specific rules
+    {
+      if: { required: ['wait'] },
+      then: {
+        properties: {
+          setVars: waitSetVarsSchema,
+        },
+        not: { required: ['expect'] },
+      },
+    },
+    // HTTP body comparison: no expect, no setVars
+    {
+      if: { required: ['httpBodyComparison'] },
+      then: {
+        not: { anyOf: [{ required: ['expect'] }, { required: ['setVars'] }] },
+      },
+    },
+    // setVars requires expect (for http and command)
+    {
+      if: {
+        required: ['setVars'],
+        anyOf: [{ required: ['http'] }, { required: ['command'] }],
+      },
+      then: {
+        required: ['expect'],
+      },
+    },
+  ],
+  additionalProperties: false,
+};
+
+const rootSchema = {
+  type: 'array',
+  items: testDefinitionSchema,
+  minItems: 1,
+};
+
+// ── Compile schema ───────────────────────────────────────────────────
+
+const ajv = new Ajv({ allErrors: true, verbose: true });
+ajvErrors(ajv);
+
+const validate = ajv.compile(rootSchema);
+
+// ── Error formatting ─────────────────────────────────────────────────
+
+/**
+ * Format Ajv validation errors into human-readable messages.
+ * @param {Array} errors - Ajv error objects
+ * @param {Array} definitions - The original test definitions array
+ * @returns {string} - Formatted error message
+ */
+function formatValidationErrors(errors, definitions) {
+  // Deduplicate and filter out noise from oneOf/anyOf wrappers
+  const seen = new Set();
+  const meaningful = [];
+
+  for (const err of errors) {
+    // Skip generic wrapper messages that aren't actionable
+    if (err.keyword === 'if' || err.keyword === 'ifThen') continue;
+
+    const key = `${err.instancePath}|${err.keyword}|${err.message}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    meaningful.push(err);
+  }
+
+  const lines = [];
+
+  for (const err of meaningful) {
+    // Extract test index from path like /0/http/method
+    const pathMatch = err.instancePath.match(/^\/(\d+)(\/.*)?$/);
+    let prefix;
+    if (pathMatch) {
+      const idx = parseInt(pathMatch[0].split('/')[1], 10);
+      const name = definitions[idx]?.name;
+      const subPath = pathMatch[2] || '';
+      prefix = name ? `Test #${idx + 1} ("${name}")` : `Test #${idx + 1}`;
+      prefix += subPath ? ` ${subPath}` : '';
+    } else {
+      prefix = err.instancePath || '(root)';
+    }
+
+    let message;
+    switch (err.keyword) {
+      case 'oneOf': {
+        // Check context: is this about test type selection?
+        if (err.instancePath.match(/^\/\d+$/) || err.instancePath === '') {
+          message = 'must define exactly one of: http, command, wait, httpBodyComparison';
+        } else {
+          message = err.message;
+        }
+        break;
+      }
+      case 'enum':
+        message = `must be one of: ${err.params.allowedValues.join(', ')}`;
+        break;
+      case 'required':
+        message = `missing required property "${err.params.missingProperty}"`;
+        break;
+      case 'additionalProperties':
+        message = `unknown property "${err.params.additionalProperty}"`;
+        break;
+      case 'type':
+        message = `must be ${err.params.type}`;
+        break;
+      case 'anyOf':
+        // Often comes from metadata needing name or labels
+        if (err.instancePath.includes('metadata')) {
+          message = 'must have either "name" or "labels"';
+        } else {
+          message = err.message;
+        }
+        break;
+      case 'not':
+        // Comes from forbidden combinations
+        if (err.instancePath.match(/^\/\d+$/)) {
+          message = 'must define exactly one of: http, command, wait, httpBodyComparison';
+        } else {
+          message = err.message;
+        }
+        break;
+      default:
+        message = err.message;
+    }
+
+    lines.push(`  ${prefix}: ${message}`);
+  }
+
+  // Deduplicate final lines
+  const uniqueLines = [...new Set(lines)];
+  return `Validation failed:\n\n${uniqueLines.join('\n')}`;
+}
+
+// ── Public API ───────────────────────────────────────────────────────
+
+/**
+ * Validate an array of parsed test definitions against the schema.
+ * Throws an Error with a descriptive multi-line message if validation fails.
+ *
+ * @param {Array<object>} definitions - Array of parsed test definition objects
+ * @throws {Error} If any definition fails validation
+ */
+function validateTestDefinitions(definitions) {
+  const valid = validate(definitions);
+  if (!valid) {
+    const message = formatValidationErrors(validate.errors, definitions);
+    throw new Error(message);
+  }
+}
+
+module.exports = { validateTestDefinitions };