testaro 78.0.8 → 78.6.0

package/validation/executors/netWatch.js

@@ -1,5 +1,7 @@
 /*
   © 2022–2024 CVS Health and/or one of its affiliates. All rights reserved.
+  © 2026 Jonathan Robert Pool.
+  © 2026 Jeff Witt.
 
   Licensed under the MIT License. See LICENSE file at the project root or
   https://opensource.org/license/mit/ for details.
@@ -8,119 +10,207 @@
 */
 
 /*
-  watchNet.js
-  Validator for network watching.
+  netWatch.js
+  Validator for network watching, covering all three auth types.
 */
 
 // IMPORTS
 
 const fs = require('fs/promises');
+const client = require('http');
 
 // CONSTANTS
 
-// Override netWatch environment variables with validation-specific ones.
 const jobDir = `${__dirname}/../jobs/todo`;
-process.env.JOB_URLS = 'http://localhost:3007/api/job';
-process.env.AGENT = 'testarauth';
-const {netWatch} = require('../../netWatch');
-const client = require('http');
 const jobID = '240101T1200-simple-example';
+const workerID = 'testaro1';
+const workerSecret = 'testarosecret';
+// Override netWatch environment variables with validation-specific ones.
+process.env.NETWATCH_URL_JOB = 'http://localhost:3007/api/job';
+process.env.NETWATCH_URL_REPORT = 'http://localhost:3007/api/report';
+process.env.NETWATCH_WORKER_ID = workerID;
+process.env.NETWATCH_WORKER_SECRET = workerSecret;
+const {netWatch} = require('../../netWatch');
+/*
+  The authorization-header value netWatch is expected to send in header mode: per RFC 7617, Basic
+  followed by base64 of testaro1:testarosecret. Intentionally a literal, not derived from the
+  netWatch code, so an encoding regression there fails here.
+*/
+const workerAuth = 'Basic dGVzdGFybzE6dGVzdGFyb3NlY3JldA==';
+// Auth types to be validated.
+const authTypes = ['header', 'pathBody', 'none'];
 
-// OPERATION
+// FUNCTIONS
 
-// Start a timer.
-const startTime = Date.now();
-// Initialize the state.
-let jobGiven = false;
-// Start checking for jobs every 5 seconds in 5 seconds.
-setTimeout(() => {
-  netWatch(false, 5, false);
-}, 5000);
-let server;
-// Handles Testaro requests to the server.
-const requestHandler = (request, response) => {
-  const {method} = request;
-  const bodyParts = [];
-  request.on('error', err => {
-    console.error(err);
-  })
-  .on('data', chunk => {
-    bodyParts.push(chunk);
-  })
-  .on('end', async () => {
-    // Remove any trailing slash from the URL.
-    const requestURL = request.url.replace(/\/$/, '');
-    // If the request method is GET:
-    if (method === 'GET') {
-      // If a job is validly requested:
-      console.log('Server got a job request from Testaro');
-      if (requestURL === `/api/job?agent=${process.env.AGENT}`) {
-        // If at least 7 seconds has elapsed since timing started:
-        if (Date.now() > startTime + 7000) {
-          // Respond with a job.
-          const jobJSON = await fs.readFile(`${jobDir}/${jobID}.json`);
-          await response.end(jobJSON);
-          console.log('Server sent job to Testaro');
-          jobGiven = true;
+// Returns an error description if a request carries wrong credentials for an auth type, else ''.
+const credentialError = (authType, request, bodyObj) => {
+  const authHeader = request.headers.authorization;
+  if (authType === 'header') {
+    if (authHeader !== workerAuth) {
+      return 'authorization header missing or wrong';
+    }
+    if (bodyObj.agentPW !== undefined) {
+      return 'agentPW unexpectedly in the request body';
+    }
+  }
+  else if (authType === 'pathBody') {
+    if (authHeader) {
+      return 'authorization header unexpectedly present';
+    }
+    if (bodyObj.agentPW !== workerSecret) {
+      return 'agentPW missing from or wrong in the request body';
+    }
+  }
+  else {
+    if (authHeader) {
+      return 'authorization header unexpectedly present';
+    }
+    if (bodyObj.agentPW !== undefined) {
+      return 'agentPW unexpectedly in the request body';
+    }
+  }
+  return '';
+};
+// Validates netWatch with one auth type; returns whether the scenario succeeded.
+const runScenario = authType => {
+  return new Promise(resolveScenario => {
+    process.env.NETWATCH_AUTH_TYPE = authType;
+    let jobGiven = false;
+    let noJobSent = false;
+    let reportOK = false;
+    let server;
+    // Responds to a request with a JSON object and a status code.
+    const respondWithObject = (object, response, statusCode = 200) => {
+      response.statusCode = statusCode;
+      response.setHeader('content-type', 'application/json; charset=utf-8');
+      response.end(JSON.stringify(object));
+    };
+    // Handles Testaro requests to the server.
+    const requestHandler = (request, response) => {
+      const {method} = request;
+      // Ignore any trailing slash on the request URL.
+      const url = request.url.replace(/\/$/, '');
+      const bodyParts = [];
+      request.on('error', err => {
+        console.error(err);
+      })
+      .on('data', chunk => {
+        bodyParts.push(chunk);
+      })
+      .on('end', async () => {
+        let bodyObj = {};
+        try {
+          bodyObj = JSON.parse(bodyParts.join('') || '{}');
         }
-        // Otherwise, i.e. if timing started less than 7 seconds ago:
-        else {
-          // Send an empty-object response.
-          await response.end('{}');
+        catch(error) {
+          console.log(`Scenario ${authType} failure: non-JSON request body (${error.message})`);
         }
-      }
-      else {
-        const error = {
-          error: 'ERROR: Job request invalid'
-        };
-        const errorJSON = JSON.stringify(error);
-        await response.end(errorJSON);
-      }
-    }
-    // Otherwise, if the request method is POST:
-    else if (method === 'POST') {
-      console.log('Server got report from Testaro');
-      const ack = {};
-      // If a report is validly submitted:
-      if (requestURL === '/api') {
-        // If a job was earlier given to Testaro:
-        if (jobGiven) {
-          // Respond, reporting success or failure.
-          try {
-            const bodyJSON = bodyParts.join('');
-            const body = JSON.parse(bodyJSON);
+        // If the request carries wrong credentials for this auth type:
+        const credError = credentialError(authType, request, bodyObj);
+        if (credError) {
+          // Report this and reject the request; netWatch is expected to abort on this.
+          console.log(`Scenario ${authType} failure: ${credError}`);
+          respondWithObject({error: 'ERROR: Authorization invalid'}, response, 401);
+          server.close();
+          return;
+        }
+        // Otherwise, if the request is a job request:
+        if (method === 'POST' && url === '/api/job') {
+          console.log(`Server got a job request from Testaro (${authType})`);
+          // If this is the first job request of the scenario:
+          if (! noJobSent) {
+            // Send a no-job response, to validate the no-job branch.
+            noJobSent = true;
+            respondWithObject({}, response);
+          }
+          // Otherwise, i.e. if the no-job branch was already validated:
+          else {
+            // Respond with a job.
+            const jobJSON = await fs.readFile(`${jobDir}/${jobID}.json`);
+            response.setHeader('content-type', 'application/json; charset=utf-8');
+            response.end(jobJSON);
+            console.log(`Server sent job to Testaro (${authType})`);
+            jobGiven = true;
+          }
+        }
+        // Otherwise, if the request is a report submission:
+        else if (method === 'POST' && url === '/api/report') {
+          console.log(`Server got report from Testaro (${authType})`);
+          const ack = {};
+          // If a job was earlier given to Testaro:
+          if (jobGiven) {
+            // Check the report, including the worker identification.
+            const {report} = bodyObj;
             if (
-              body.acts
-              && body.sources
-              && body.sources.agent
-              && body.sources.agent === process.env.AGENT
+              report
+              && report.acts
+              && report.jobData
+              && report.sources
+              && report.sources.agent === workerID
             ) {
               ack.message = 'Success: Valid report submitted';
+              reportOK = true;
             }
             else {
-              ack.message = 'Failure: Report invalid';
+              ack.message = 'Failure: Report invalid or worker not identified';
             }
           }
-          catch(error) {
-            ack.message = `ERROR: ${error.message}`;
+          else {
+            ack.message = 'ERROR: Report submitted before a job was given';
           }
+          respondWithObject(ack, response);
+          console.log(`Server responded: ${ack.message}`);
+          // This ends the scenario, so stop the server.
+          server.close();
+          console.log('Server closed');
         }
-      }
-      else {
-        ack.message = 'ERROR: Report submission invalid';
-      }
-      const ackJSON = JSON.stringify(ack);
-      response.end(ackJSON);
-      console.log(`Server responded: ${ack.message}`);
-      // This ends the validation, so stop the server.
+        // Otherwise, i.e. if the request is neither:
+        else {
+          // Report this and reject the request.
+          console.log(`Scenario ${authType} failure: unexpected request ${method} ${request.url}`);
+          respondWithObject({error: 'ERROR: Request invalid'}, response, 404);
+          server.close();
+        }
+      });
+    };
+    // Create a server.
+    server = client.createServer({}, requestHandler);
+    // Start the server listening for Testaro requests, then start a one-job watch.
+    server.listen(3007, async () => {
+      console.log(`Job and report server listening on port 3007 (auth type ${authType})`);
+      const watchOK = await netWatch(false, 5, false);
       server.close();
-      console.log('Server closed');
-    }
+      resolveScenario(reportOK && watchOK);
+    });
   });
 };
-// Create a server.
-server = client.createServer({}, requestHandler);
-// Start a server listening for Testaro requests.
-server.listen(3007, () => {
-  console.log('Job and report server listening on port 3007');
-});
+
+// OPERATION
+
+// Impose a deadline, so a hang becomes a failure instead of a silent stall.
+const watchdog = setTimeout(() => {
+  console.log('Failure: netWatch validation timed out');
+  process.exit(1);
+}, 600000);
+// Run the scenarios in sequence and exit with a code reporting the outcome.
+(async () => {
+  const failures = [];
+  for (const authType of authTypes) {
+    const isOK = await runScenario(authType);
+    // Drop any kept-alive connections to the closed server before the next scenario.
+    client.globalAgent.destroy();
+    console.log(`Scenario ${authType}: ${isOK ? 'valid' : 'INVALID'}\n`);
+    if (! isOK) {
+      failures.push(authType);
+    }
+  }
+  clearTimeout(watchdog);
+  if (failures.length) {
+    console.log(`Failure: Invalid scenarios: ${failures.join(', ')}`);
+    process.exit(1);
+  }
+  else {
+    console.log('Success: Valid reports submitted for all auth types');
+    process.exit(0);
+  }
+})();

package/validation/executors/test.js

@@ -1,5 +1,6 @@
 /*
   © 2022–2023 CVS Health and/or one of its affiliates. All rights reserved.
+  © 2026 Jeff Witt.
 
   Licensed under the MIT License. See LICENSE file at the project root or
   https://opensource.org/license/mit/ for details.
@@ -13,6 +14,20 @@
   Execution example: npm test focOp
 */
 
-const testID = process.argv[2];
 const {validateTest} = require('../validateTest');
-validateTest(testID);
+
+// Flushes the standard output and then exits, because lingering browser
+// processes otherwise keep the event loop alive indefinitely.
+const exitWith = code => {
+  process.stdout.write('', () => process.exit(code));
+};
+
+const testID = process.argv[2];
+validateTest(testID)
+.then(result => {
+  exitWith(result.success ? 0 : 1);
+})
+.catch(error => {
+  console.log(`ERROR validating test ${testID} (${error.message})`);
+  exitWith(1);
+});

package/validation/executors/tests.js

@@ -1,5 +1,6 @@
 /*
   © 2022–2024 CVS Health and/or one of its affiliates. All rights reserved.
+  © 2026 Jeff Witt.
 
   Licensed under the MIT License. See LICENSE file at the project root or
   https://opensource.org/license/mit/ for details.
@@ -14,21 +15,128 @@
 
 // IMPORTS
 
+require('dotenv').config({quiet: true});
 const fs = require('fs').promises;
-const {validateTest} = require('../validateTest');
+const {spawn} = require('child_process');
+
+// CONSTANTS
+
+const ruleDir = `${__dirname}/../../testaro`;
+const validatorDir = `${__dirname}/../tests/jobProperties`;
+// Rules that ask an AI model to classify candidate instances.
+const aiRuleIDs = ['allCaps'];
+// Validators of features other than rules, run after the rule validators.
+const featureValidatorIDs = ['checkpoint', 'checkpoint-browser', 'checkpoint-page', 'userPath'];
+// Maximum number of seconds allowed for the validation of one rule.
+const ruleTimeLimit = 180;
+/*
+  Rules whose validations currently fail, with the reasons. They do not make
+  the validation as a whole fail, so a failure of a healthy rule is blocking
+  while these known failures are being repaired. When a repair makes a rule
+  pass, it is to be removed from knownFailures.json.
+*/
+const knownFailures = require('../knownFailures.json');
+
+// FUNCTIONS
+
+/*
+  Validates one rule in a child process, so a defective rule cannot crash or
+  hang the whole validation and so each job's browser processes are reaped.
+  Returns whether the validation succeeded.
+*/
+const validateInChild = testID => new Promise(resolve => {
+  const child = spawn(
+    process.execPath, [`${__dirname}/test.js`, testID], {stdio: 'inherit'}
+  );
+  const timer = setTimeout(() => {
+    console.log(`######## Failure: Validation of ${testID} timed out (${ruleTimeLimit}s)\n`);
+    child.kill('SIGKILL');
+  }, 1000 * ruleTimeLimit);
+  child.on('close', code => {
+    clearTimeout(timer);
+    resolve(code === 0);
+  });
+  child.on('error', error => {
+    clearTimeout(timer);
+    console.log(`######## Failure: Validation of ${testID} could not start (${error.message})\n`);
+    resolve(false);
+  });
+});
 
 // OPERATION
 
-// Get the names of the Testaro tes files.
-fs.readdir(`${__dirname}/../../testaro/`)
+// Get the names of the Testaro rule files and the validator files.
+Promise.all([fs.readdir(ruleDir), fs.readdir(validatorDir)])
 // When they arrive:
-.then(async fileNames => {
-  // For each file name:
-  for (const fileName of fileNames) {
-    // Get the test ID from it by disregarding its extension.
-    const testID = fileName.replace(/\..+$/, '');
-    // Validate the test.
+.then(async ([ruleFileNames, validatorFileNames]) => {
+  // Get the rule IDs and the IDs of the rules with validators.
+  // Rule modules are the JavaScript files in the rule directory, except the generated registry.
+  const ruleIDs = ruleFileNames
+  .filter(name => name.endsWith('.js') && name !== 'registry.js')
+  .map(name => name.slice(0, -3));
+  const validatorIDs = validatorFileNames
+  .filter(name => name.endsWith('.json'))
+  .map(name => name.slice(0, -5));
+  // Identify the rules without validators and the validators without rules.
+  const unvalidatedIDs = ruleIDs.filter(id => ! validatorIDs.includes(id));
+  const orphanIDs = validatorIDs.filter(
+    id => ! ruleIDs.includes(id) && ! featureValidatorIDs.includes(id)
+  );
+  // Identify the AI-dependent rules to be skipped for lack of an API key.
+  const skippedIDs = process.env.ANTHROPIC_API_KEY
+    ? []
+    : ruleIDs.filter(id => aiRuleIDs.includes(id));
+  // Initialize the results.
+  const failedIDs = [];
+  const knownFailedIDs = [];
+  const recoveredIDs = [];
+  let validatedCount = 0;
+  // For each rule with a validator, and then each feature validator:
+  const featureIDs = featureValidatorIDs.filter(id => validatorIDs.includes(id));
+  for (
+    const testID of ruleIDs
+    .filter(id => validatorIDs.includes(id) && ! skippedIDs.includes(id))
+    .concat(featureIDs)
+  ) {
+    // Validate the rule.
     console.log(`### Validating Testaro test ${testID}`);
-    await validateTest(testID);
+    const succeeded = await validateInChild(testID);
+    validatedCount++;
+    if (succeeded && knownFailures[testID]) {
+      recoveredIDs.push(testID);
+    }
+    else if (! succeeded) {
+      (knownFailures[testID] ? knownFailedIDs : failedIDs).push(testID);
+    }
+  }
+  // Report a summary.
+  console.log('\n#### Validation summary');
+  console.log(`Rules validated: ${validatedCount}`);
+  console.log(`Rules failing validation: ${failedIDs.length + knownFailedIDs.length}`);
+  if (failedIDs.length) {
+    console.log(`  Failing rules: ${failedIDs.join(', ')}`);
+  }
+  if (knownFailedIDs.length) {
+    console.log(`  Known-failing rules, not blocking: ${knownFailedIDs.join(', ')}`);
+  }
+  if (recoveredIDs.length) {
+    console.log(
+      `Warning - rules now passing; remove them from knownFailures.json: ${recoveredIDs.join(', ')}`
+    );
+  }
+  if (skippedIDs.length) {
+    console.log(
+      `Warning - AI-dependent rules skipped because ANTHROPIC_API_KEY is not set: ${
+        skippedIDs.join(', ')
+      }`
+    );
+  }
+  if (unvalidatedIDs.length) {
+    console.log(`Warning - rules without validators: ${unvalidatedIDs.join(', ')}`);
+  }
+  if (orphanIDs.length) {
+    console.log(`Warning - validators without rules: ${orphanIDs.join(', ')}`);
   }
+  // Make the validation failable by CI and other callers.
+  process.exitCode = failedIDs.length ? 1 : 0;
 });