testomatio-reporter-cli 2.15.0-beta.1-json-output → 2.16.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testomatio-reporter-cli",
-  "version": "2.15.0-beta.1-json-output",
+  "version": "2.16.0",
   "description": "Yarn Berry compatible standalone CLI for @testomatio/reporter",
   "type": "module",
   "bin": {

package/src/adapter/codecept.js

@@ -34,7 +34,8 @@ const HOOK_EXECUTION_ORDER = {
 // codeceptjs workers are self-contained
 dataStorage.isFileStorage = false;
 
-const DATA_REGEXP = /[|\s]+?(\{".*\}|\[.*\])/;
+// CodeceptJS appends the serialized data row of a data-driven test to its title
+const DATA_REGEXP = / \| (\{.*\}|\[.*\]|null|"(?:\\.|[^"\\])*")((?:\s+@[a-zA-Z0-9-_]+)*)$/;
 
 if (MAJOR_VERSION < 3) {
   console.log('🔴 This reporter works with CodeceptJS 3+, please update your tests');
@@ -301,7 +302,7 @@ function CodeceptReporter(config) {
   });
 
   event.dispatcher.on(event.step.started, step => {
-    const stepText = `${repeat(output.stepShift)} ${step.toCliStyled ? step.toCliStyled() : step.toString()}`;
+    const stepText = `${repeat(output.stepShift)} ${formatStepLog(step)}`;
     dataStorage.putData('log', stepText);
   });
 
@@ -349,15 +350,30 @@ function stripExampleFromTitle(title) {
   const res = title.match(DATA_REGEXP);
   if (!res) return { title, example: null };
 
+  let example = null;
+  let exampleParsed = false;
   try {
-    const example = JSON.parse(res[1]);
-    title = title.replace(DATA_REGEXP, '').trim();
-    return { title, example };
+    example = JSON.parse(res[1]);
+    exampleParsed = true;
   } catch (e) {
-    // If JSON parsing fails, return title without example
-    debug('Failed to parse example JSON:', res[1], e.message);
-    return { title: title.replace(DATA_REGEXP, '').trim(), example: null };
+    try {
+      example = JSON.parse(res[1].slice(1, -1));
+      exampleParsed = true;
+    } catch (e2) {
+      debug('Failed to parse example from title:', res[1]);
+    }
+  }
+
+  let baseTitle = title.slice(0, res.index).trim();
+  if (exampleParsed && baseTitle.includes('${current}')) {
+    baseTitle = baseTitle.replaceAll('${current}', formatExample(example));
   }
+  return { title: `${baseTitle}${res[2]}`, example };
+}
+
+function formatExample(example) {
+  if (example !== null && typeof example === 'object') return JSON.stringify(example);
+  return String(example);
 }
 
 function stripTagsFromTitle(title) {
@@ -369,6 +385,18 @@ function repeat(num) {
   return ''.padStart(num, ' ');
 }
 
+function formatStepLog(step) {
+  if (typeof step?.toCliStyled === 'function') return step.toCliStyled();
+
+  if (typeof step?.toString === 'function' && step.toString !== Object.prototype.toString) {
+    return step.toString();
+  }
+
+  const title = step?.title || '';
+  const args = Array.isArray(step?.args) ? step.args.join(', ') : '';
+  return `${title}${args ? ` ${args}` : ''}`.trim();
+}
+
 // Helper functions for cleaner event handling
 function initializeTestDataStore() {
   if (!global.testomatioDataStore) global.testomatioDataStore = {};

package/src/bin/cli.js

@@ -43,6 +43,8 @@ program
     if (subOpts.filterList || subOpts.format) {
       process.env.TESTOMATIO_LOG_STDERR = '1';
       process.env.TESTOMATIO_LOG_LEVEL ||= 'WARN';
+      // with --format json the logs are machine-readable too: one JSON object per line on stderr
+      if (subOpts.format === 'json') process.env.TESTOMATIO_LOG_JSON = '1';
     } else {
       console.log(pc.cyan(pc.bold(` 🤩 Testomat.io Reporter v${version}`)));
     }
@@ -82,7 +84,7 @@ program
 
     await client.createRun(createRunParams);
 
-    const runId = client.pipeStore.runId || process.env.runId;
+    const runId = client.pipeStore.runId;
     if (!runId) {
       log.error(pc.red('Failed to create run on Testomat.io.'));
       process.exit(1);
@@ -94,7 +96,7 @@ program
     await client.updateRunStatus('pending', { tests: plannedTests });
 
     // stdout carries ONLY the run data so it can be captured: RUN_ID=$(reporter start)
-    console.log(formatRunOutput({ ...client.pipeStore, runId }, opts.format));
+    console.log(formatRunOutput(client.pipeStore, opts.format));
     process.exit(0);
   });
 
@@ -250,15 +252,20 @@ program
 
       if (apiKey) {
         await client.createRun(createRunParams);
-        const runId = process.env.TESTOMATIO_RUN || process.env.runId;
+
+        const runId = client.pipeStore.runId;
+        if (!runId) {
+          log.error(pc.red('Failed to create run on Testomat.io.'));
+          process.exit(1);
+        }
+
         if (client.pipeStore.runUrl) log.info( `📊 Report URL: ${pc.magenta(client.pipeStore.runUrl)}`);
 
         if (opts.kind !== 'manual') {
           log.info( `No command passed, so you need to run tests yourself:`);
           log.info( `TESTOMATIO_RUN=${runId} <command>`);
         }
-        const runOutput = formatRunOutput({ ...client.pipeStore, runId }, opts.format);
-        if (opts.format && runOutput) console.log(runOutput);
+        if (opts.format) console.log(formatRunOutput(client.pipeStore, opts.format));
       } else {
         log.info( '⚠️  No API key provided. Cannot create run without TESTOMATIO key.');
         process.exit(1);

package/src/client.js

@@ -187,7 +187,7 @@ class Client {
       }
 
     } catch (err) {
-      console.error(APP_PREFIX, 'Error in uploadStepArtifacts for testRid', testRid, ':', err);
+      log.error('Error in uploadStepArtifacts for testRid', testRid, ':', err.message || err);
       throw err;
     }
   }
@@ -225,7 +225,7 @@ class Client {
     try {
       await this.uploadStepArtifacts(steps, rid);
     } catch (err) {
-      console.log(APP_PREFIX, 'Failed to upload step artifacts:', err);
+      log.error('Failed to upload step artifacts:', err.message || err);
     }
 
     const uploadedFiles = [];
@@ -423,7 +423,7 @@ class Client {
           const pathPadding = Math.max(...failedUploads.map(upload => upload.relativePath.length)) + 1;
 
           failedUploads.forEach(upload => {
-            console.log(
+            log.info(
               `  ${pc.gray('|')} 🔴 ${upload.relativePath.padEnd(pathPadding)} ${pc.gray(
                 `| ${upload.sizePretty.padStart(filesizeStrMaxLength)} |`,
               )}`,
@@ -439,7 +439,7 @@ class Client {
           }));
           const pathPadding = Math.max(...skippedUploads.map(upload => upload.relativePath.length)) + 1;
           skippedUploads.forEach(upload => {
-            console.log(
+            log.info(
               `  ${pc.gray('|')} 🟡 ${upload.relativePath.padEnd(pathPadding)} ${pc.gray(
                 `| ${upload.sizePretty.padStart(filesizeStrMaxLength)} |`,
               )}`,

package/src/pipe/csv.js

@@ -6,6 +6,7 @@ import pc from 'picocolors';
 import merge from 'lodash.merge';
 import { isSameTest, getCurrentDateTime, ansiRegExp } from '../utils/utils.js';
 import { CSV_HEADERS } from '../constants.js';
+import { log } from '../utils/log.js';
 
 const debug = createDebugMessages('@testomatio/reporter:pipe:csv');
 /**
@@ -76,11 +77,11 @@ class CsvPipe {
     this.checkExportDir();
 
     if (!this.outputFile) {
-      console.log(pc.yellow(`⚠️  CSV file is not set, ignoring`));
+      log.warn(pc.yellow(`⚠️  CSV file is not set, ignoring`));
       return;
     }
 
-    console.log(pc.yellow(`⏳ The test results will be added to the csv. It will take some time...`));
+    log.info(pc.yellow(`⏳ The test results will be added to the csv. It will take some time...`));
 
     try {
       // Create csv writer object
@@ -91,7 +92,7 @@ class CsvPipe {
       // Save csv file based on the current data
       return await writer.writeRecords(data);
     } catch (e) {
-      console.log('Unknown csv error: ', e);
+      log.error('Unknown csv error: ', e);
     }
   }
 
@@ -135,7 +136,7 @@ class CsvPipe {
     // Save results based on the default headers
     if (this.isEnabled) {
       await this.saveToCsv(this.results, CSV_HEADERS);
-      console.log(pc.green(`🗃️  Recording completed! You can check the result in file = ${this.outputFile}`));
+      log.info(pc.green(`🗃️  Recording completed! You can check the result in file = ${this.outputFile}`));
     }
   }
 

package/src/pipe/html.js

@@ -8,6 +8,7 @@ import { marked } from 'marked';
 import fileUrl from 'file-url';
 import { fileSystem, isSameTest, ansiRegExp, formatStep, transformEnvVarToBoolean } from '../utils/utils.js';
 import { HTML_REPORT } from '../constants.js';
+import { log } from '../utils/log.js';
 import { fileURLToPath } from 'node:url';
 
 const debug = createDebugMessages('@testomatio/reporter:pipe:html');
@@ -144,14 +145,14 @@ class HtmlPipe {
     debug('HTML tests data:', tests);
 
     if (!outputPath) {
-      console.log(pc.yellow(`🚨 HTML export path is not set, ignoring...`));
+      log.warn(pc.yellow(`🚨 HTML export path is not set, ignoring...`));
       return;
     }
 
-    console.log(pc.yellow(`⏳ The test results will be added to the HTML report. It will take some time...`));
+    log.info(pc.yellow(`⏳ The test results will be added to the HTML report. It will take some time...`));
 
     if (msg) {
-      console.log(pc.blue(msg));
+      log.info(pc.blue(msg));
     }
 
     const aggregatedTests = aggregateTestRetries(tests);
@@ -295,9 +296,9 @@ class HtmlPipe {
 
       debug('HTML tests data:', fileUrlPath);
 
-      console.log(pc.green(`📊 The HTML report was successfully generated. Full filepath: ${fileUrlPath}`));
+      log.info(pc.green(`📊 The HTML report was successfully generated. Full filepath: ${fileUrlPath}`));
     } else {
-      console.log(pc.red(`🚨 Failed to generate the HTML report.`));
+      log.error(pc.red(`🚨 Failed to generate the HTML report.`));
     }
   }
 
@@ -309,7 +310,7 @@ class HtmlPipe {
    */
   #generateHTMLReport(data, templatePath = '') {
     if (!templatePath) {
-      console.log(pc.red(`🚨 HTML template not found. Report generation is impossible!`));
+      log.error(pc.red(`🚨 HTML template not found. Report generation is impossible!`));
       return;
     }
 
@@ -320,8 +321,8 @@ class HtmlPipe {
 
       return template(data);
     } catch (e) {
-      console.log(pc.red('❌ Oops! An unknown error occurred when generating an HTML report'));
-      console.log(pc.red(e));
+      log.error(pc.red('❌ Oops! An unknown error occurred when generating an HTML report'));
+      log.error(pc.red(e));
     }
   }
 

package/src/pipe/testomatio.js

@@ -18,6 +18,7 @@ import {
   getGitCommitSha,
 } from '../utils/utils.js';
 import { parseFilterParams, generateFilterRequestParams, setS3Credentials } from '../utils/pipe_utils.js';
+import { hideTestomatioToken } from '../utils/hide_token.js';
 import { config } from '../config.js';
 import { log } from '../utils/log.js';
 
@@ -87,12 +88,15 @@ class TestomatioPipe {
     this.store = store || {};
     this.title = params.title || process.env.TESTOMATIO_TITLE;
     this.sharedRun = !!process.env.TESTOMATIO_SHARED_RUN;
+    this.sharedRunShards = process.env.TESTOMATIO_SHARDS
+      ? parseInt(process.env.TESTOMATIO_SHARDS, 10) || undefined
+      : undefined;
     this.sharedRunTimeout = process.env.TESTOMATIO_SHARED_RUN_TIMEOUT
       ? parseInt(process.env.TESTOMATIO_SHARED_RUN_TIMEOUT, 10)
       : undefined;
 
-    if (this.sharedRunTimeout && !this.sharedRun) {
-      debug('Auto-enabling sharedRun because sharedRunTimeout is set');
+    if ((this.sharedRunTimeout || this.sharedRunShards) && !this.sharedRun) {
+      debug('Auto-enabling sharedRun because sharedRunTimeout or sharedRunShards is set');
       this.sharedRun = true;
     }
 
@@ -322,6 +326,7 @@ class TestomatioPipe {
         label: this.label,
         shared_run: this.sharedRun,
         shared_run_timeout: this.sharedRunTimeout,
+        shared_run_shards: this.sharedRunShards,
         kind: params.kind,
         status: params.status,
         configuration,
@@ -380,16 +385,15 @@ class TestomatioPipe {
       process.env.runId = this.runId;
       debug('Run created', this.runId);
     } catch (err) {
-      if (!this.apiKey) console.error('Testomat.io API key is not set');
+      if (!this.apiKey) log.error('Testomat.io API key is not set');
       const errorText = err.response?.data?.message || err.message;
       debug('Error creating run', err);
-      console.log(APP_PREFIX, errorText || err);
+      log.error(errorText || err);
       if (err.response?.status === 403) this.#disablePipe();
 
       this.#logFailedResponse(err);
 
-      console.error(
-        APP_PREFIX,
+      log.error(
         'Error creating Testomat.io report (see details above), please check if your API key is valid. Skipping report',
       );
       printCreateIssue();
@@ -409,7 +413,7 @@ class TestomatioPipe {
       this.reportingCanceledDueToReqFailures = true;
       let errorMessage = `⚠️ ${process.env.TESTOMATIO_MAX_REQUEST_FAILURES}`;
       errorMessage += ' requests were failed, reporting to Testomat aborted.';
-      console.warn(`${APP_PREFIX} ${pc.yellow(errorMessage)}`);
+      log.warn(pc.yellow(errorMessage));
     }
     return cancelReporting;
   }
@@ -557,7 +561,7 @@ class TestomatioPipe {
       const errorMessage = pc.red(
         `⚠️ Due to request failures, ${this.notReportedTestsCount} test(s) were not reported to Testomat.io`,
       );
-      console.warn(`${APP_PREFIX} ${errorMessage}`);
+      log.warn(errorMessage);
     }
 
     const { status } = params;
@@ -570,6 +574,7 @@ class TestomatioPipe {
     if (status === STATUS.FINISHED) status_event = 'finish';
     if (status === STATUS.PASSED) status_event = 'pass';
     if (status === STATUS.FAILED) status_event = 'fail';
+    if (this.sharedRun) status_event = 'finish';
 
     try {
       if (this.runId && !this.proceed) {
@@ -621,7 +626,7 @@ class TestomatioPipe {
         );
       }
     } catch (err) {
-      console.log(APP_PREFIX, 'Error updating status, skipping...', err);
+      log.error('Error updating status, skipping...', err.message || err);
       this.#logFailedResponse(err);
       printCreateIssue();
     }
@@ -665,18 +670,32 @@ class TestomatioPipe {
 
     message += `\t${pc.bold('response: ')}${pc.gray(responseBody)}\n`;
 
-    const requestBody = hideTestomatioToken(stringify(error.response?.config?.data));
+    let requestBody = hideTestomatioToken(stringify(error.response?.config?.data));
+    let requestTruncated = false;
     if (process.env.DEBUG || process.env.TESTOMATIO_DEBUG || requestBody.length < 1000) {
       // full body
       message += `\t${pc.bold('request: ')}${pc.gray(requestBody)}\n`;
     } else {
       // cut body
-      const requestBodyCut = requestBody.slice(0, 1000);
-      message += `\t${pc.bold('request: ')}${pc.gray(`${requestBodyCut}...`)}\n`;
+      requestTruncated = true;
+      requestBody = `${requestBody.slice(0, 1000)}...`;
+      message += `\t${pc.bold('request: ')}${pc.gray(requestBody)}\n`;
       message += '\trequest body is cut, run with TESTOMATIO_DEBUG=1 to see full body\n';
     }
 
-    console.log(message);
+    // the JSON line is built from the same values as the text message, with the token already hidden
+    const fields = {
+      status: statusCode,
+      method,
+      url,
+      error: apiMessage || statusText || undefined,
+      response: parseIfJson(responseBody),
+      request: parseIfJson(requestBody),
+    };
+    // a cut body is no longer valid JSON, so consumers are told why `request` is a string
+    if (requestTruncated) fields.requestTruncated = true;
+
+    log.errorWithFields(fields, message);
 
     if (error.response?.data?.message?.includes('could not be matched')) {
       this.hasUnmatchedTests = true;
@@ -693,8 +712,7 @@ function printCreateIssue() {
   if (registeredErrorHints) return;
   registeredErrorHints = true;
   process.on('exit', () => {
-    console.log(
-      APP_PREFIX,
+    log.error(
       'There was an error reporting to Testomat.io.\n',
       pc.yellow(
         'If you think this is a bug please create an issue: https://github.com/testomatio/reporter/issues/new.',
@@ -704,18 +722,6 @@ function printCreateIssue() {
   });
 }
 
-/**
- * Removes Testomatio token from string data
- *
- * @param {string} data
- * @returns {string}
- */
-function hideTestomatioToken(data) {
-  return (typeof data === 'string' ? data : '')
-    .replace(/"api_key"\s*:\s*"[^"]+"/g, '"api_key": "<hidden>"')
-    .replace(/"(tstmt_[^"]+)"/g, '"tstmt_***"');
-}
-
 /**
  * Stringifies provided data
  *
@@ -727,4 +733,18 @@ function stringify(anything, opts = { pretty: false }) {
   return typeof anything === 'string' ? anything : JSON.stringify(anything, null, opts.pretty ? 2 : undefined);
 }
 
+/**
+ * Turn a JSON string back into an object for structured logs; keeps the string if it is not JSON.
+ *
+ * @param {string} data
+ * @returns {any}
+ */
+function parseIfJson(data) {
+  try {
+    return JSON.parse(data);
+  } catch {
+    return data;
+  }
+}
+
 export default TestomatioPipe;

package/src/utils/hide_token.js

@@ -0,0 +1,12 @@
+/**
+ * Hides the Testomat.io API token in any data which is about to be printed or logged.
+ * Applied at the logger level, so a raw error object with a request body can't leak the token.
+ *
+ * @param {string} data
+ * @returns {string} The data with every token replaced, empty string if data is not a string.
+ */
+export function hideTestomatioToken(data) {
+  if (typeof data !== 'string') return '';
+
+  return data.replace(/"api_key"\s*:\s*"[^"]+"/g, '"api_key": "<hidden>"').replace(/tstmt_[\w-]+/g, 'tstmt_***');
+}

package/src/utils/log.js

@@ -1,4 +1,6 @@
+import { format as formatArgs, stripVTControlCharacters } from 'util';
 import { APP_PREFIX } from '../constants.js';
+import { hideTestomatioToken } from './hide_token.js';
 
 /**
  * Log levels for the Testomat.io reporter.
@@ -35,16 +37,54 @@ export function shouldLog(messageLevel) {
   return messageLevel <= getLogLevel() || !!process.env.TESTOMATIO_DEBUG;
 }
 
+/**
+ * Check if logs should be printed as JSON lines instead of [TESTOMATIO] prefixed text.
+ * Enabled by the CLI for `--format json` so the whole output is machine-readable.
+ * @returns {boolean}
+ */
+export function isJsonOutput() {
+  return process.env.TESTOMATIO_LOG_JSON === '1';
+}
+
+/**
+ * Render a log message as a single JSON line, e.g. `{"level":"error","message":"..."}`.
+ * Colors are stripped, so the message stays readable after parsing.
+ * @param {string} level - Log level name
+ * @param {any[]} args - Arguments as passed to the log function
+ * @param {Object} [fields] - Extra fields to add to the JSON object
+ * @returns {string}
+ */
+function jsonLine(level, args, fields = {}) {
+  const message = stripVTControlCharacters(formatArgs(...args)).trim();
+  return hideTestomatioToken(JSON.stringify({ ...fields, level, message }));
+}
+
+/**
+ * Render the arguments of a log function as text, with the API token hidden.
+ * Errors and other objects are formatted the way console does it.
+ * @param {any[]} args - Arguments as passed to the log function
+ * @returns {string}
+ */
+function textLine(args) {
+  return hideTestomatioToken(formatArgs(...args));
+}
+
 /**
  * Log an info message with [TESTOMATIO] prefix.
  * Only logs when TESTOMATIO_LOG_LEVEL is INFO.
  * @param {...any} args - Arguments to log
  */
 export function info(...args) {
-  if (shouldLog(LOG_LEVELS.INFO)) {
-    const fn = process.env.TESTOMATIO_LOG_STDERR === '1' ? console.error : console.log;
-    fn(APP_PREFIX, ...args);
+  if (!shouldLog(LOG_LEVELS.INFO)) return;
+
+  let fn = console.log;
+  if (process.env.TESTOMATIO_LOG_STDERR === '1') fn = console.error;
+
+  if (isJsonOutput()) {
+    fn(jsonLine('info', args));
+    return;
   }
+  fn(APP_PREFIX, textLine(args));
 }
 
 /**
@@ -53,9 +93,13 @@ export function info(...args) {
  * @param {...any} args - Arguments to log
  */
 export function warn(...args) {
-  if (shouldLog(LOG_LEVELS.WARN)) {
-    console.warn(APP_PREFIX, ...args);
+  if (!shouldLog(LOG_LEVELS.WARN)) return;
+
+  if (isJsonOutput()) {
+    console.warn(jsonLine('warn', args));
+    return;
   }
+  console.warn(APP_PREFIX, textLine(args));
 }
 
 /**
@@ -64,9 +108,29 @@ export function warn(...args) {
  * @param {...any} args - Arguments to log
  */
 export function error(...args) {
-  if (shouldLog(LOG_LEVELS.ERROR)) {
-    console.error(APP_PREFIX, ...args);
+  if (!shouldLog(LOG_LEVELS.ERROR)) return;
+
+  if (isJsonOutput()) {
+    console.error(jsonLine('error', args));
+    return;
+  }
+  console.error(APP_PREFIX, textLine(args));
+}
+
+/**
+ * Log an error which carries structured data, e.g. a failed API request.
+ * The fields are added to the JSON line; in text mode only the message is printed.
+ * @param {Object} fields - Extra fields to add to the JSON object
+ * @param {...any} args - Arguments to log as text
+ */
+export function errorWithFields(fields, ...args) {
+  if (!shouldLog(LOG_LEVELS.ERROR)) return;
+
+  if (isJsonOutput()) {
+    console.error(jsonLine('error', args, fields));
+    return;
   }
+  console.error(APP_PREFIX, textLine(args));
 }
 
 /**
@@ -82,6 +146,8 @@ export const log = {
   info,
   warn,
   error,
+  errorWithFields,
+  isJsonOutput,
   getLogLevel,
   shouldLog,
   LOG_LEVELS,