testomatio-reporter-cli 2.14.0 → 2.15.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testomatio-reporter-cli",
-  "version": "2.14.0",
+  "version": "2.15.0",
   "description": "Yarn Berry compatible standalone CLI for @testomatio/reporter",
   "type": "module",
   "bin": {

package/src/adapter/codecept.js

@@ -301,7 +301,7 @@ function CodeceptReporter(config) {
   });
 
   event.dispatcher.on(event.step.started, step => {
-    const stepText = `${repeat(output.stepShift)} ${step.toCliStyled ? step.toCliStyled() : step.toString()}`;
+    const stepText = `${repeat(output.stepShift)} ${formatStepLog(step)}`;
     dataStorage.putData('log', stepText);
   });
 
@@ -369,6 +369,18 @@ function repeat(num) {
   return ''.padStart(num, ' ');
 }
 
+function formatStepLog(step) {
+  if (typeof step?.toCliStyled === 'function') return step.toCliStyled();
+
+  if (typeof step?.toString === 'function' && step.toString !== Object.prototype.toString) {
+    return step.toString();
+  }
+
+  const title = step?.title || '';
+  const args = Array.isArray(step?.args) ? step.args.join(', ') : '';
+  return `${title}${args ? ` ${args}` : ''}`.trim();
+}
+
 // Helper functions for cleaner event handling
 function initializeTestDataStore() {
   if (!global.testomatioDataStore) global.testomatioDataStore = {};

package/src/bin/cli.js

@@ -16,7 +16,7 @@ import { filesize as prettyBytes } from 'filesize';
 import dotenv from 'dotenv';
 import Replay from '../replay.js';
 import { log } from '../utils/log.js';
-import { formatFilterListIds } from '../utils/pipe_utils.js';
+import { formatFilterListIds, formatRunOutput } from '../utils/pipe_utils.js';
 import fs from 'fs';
 import path from 'path';
 
@@ -43,6 +43,8 @@ program
     if (subOpts.filterList || subOpts.format) {
       process.env.TESTOMATIO_LOG_STDERR = '1';
       process.env.TESTOMATIO_LOG_LEVEL ||= 'WARN';
+      // with --format json the logs are machine-readable too: one JSON object per line on stderr
+      if (subOpts.format === 'json') process.env.TESTOMATIO_LOG_JSON = '1';
     } else {
       console.log(pc.cyan(pc.bold(` 🤩 Testomat.io Reporter v${version}`)));
     }
@@ -53,7 +55,7 @@ program
   .description('Start a new run and return its ID')
   .option('--kind <type>', 'Specify run type: automated, manual, mixed, or detect')
   .option('--filter <filter>', 'Scope the prepared run to tests matching the filter (no execution)')
-  .option('--format <format>', 'Machine-readable output: print only the run id to stdout (e.g. --format id)')
+  .option('--format <format>', 'Machine-readable output: the run id (--format id) or run details (--format json)')
   .option('--warn', 'Exit 0 instead of 1 when the filter matches no tests (warn only)')
   .action(async opts => {
     cleanLatestRunId();
@@ -82,7 +84,7 @@ program
 
     await client.createRun(createRunParams);
 
-    const runId = client.pipeStore.runId || process.env.runId;
+    const runId = client.pipeStore.runId;
     if (!runId) {
       log.error(pc.red('Failed to create run on Testomat.io.'));
       process.exit(1);
@@ -93,8 +95,8 @@ program
     const plannedTests = (client.pipeStore.preparedTestIds || []).map(id => ({ test_id: id, title: id }));
     await client.updateRunStatus('pending', { tests: plannedTests });
 
-    // stdout carries ONLY the run id so it can be captured: RUN_ID=$(reporter start)
-    console.log(runId);
+    // stdout carries ONLY the run data so it can be captured: RUN_ID=$(reporter start)
+    console.log(formatRunOutput(client.pipeStore, opts.format));
     process.exit(0);
   });
 
@@ -129,7 +131,10 @@ program
   .argument('[command]', 'Test runner command')
   .option('--filter <filter>', 'Additional execution filter')
   .option('--filter-list <filter>', 'Get a list of all tests by filter before running')
-  .option('--format <format>', 'Machine-readable output format for --filter-list (grep, json, newline, ids)')
+  .option(
+    '--format <format>',
+    'Machine-readable output: test ids for --filter-list (grep, json, newline, ids), or the run created (id, json)',
+  )
   .option('--kind <type>', 'Specify run type: automated, manual, mixed, or detect')
   .option('--remote <profile>', 'Trigger run on the named Testomat.io CI profile instead of executing locally')
   .option(
@@ -230,6 +235,8 @@ program
 
       log.info(`🚀 CI build triggered on profile ${pc.cyan(opts.remote)}`);
       log.info(`📊 Report URL: ${pc.magenta(client.pipeStore.runUrl)}`);
+      const remoteOutput = formatRunOutput(client.pipeStore, opts.format);
+      if (opts.format && remoteOutput) console.log(remoteOutput);
       return process.exit(0);
     }
 
@@ -245,13 +252,20 @@ program
 
       if (apiKey) {
         await client.createRun(createRunParams);
-        const runId = process.env.TESTOMATIO_RUN || process.env.runId;
+
+        const runId = client.pipeStore.runId;
+        if (!runId) {
+          log.error(pc.red('Failed to create run on Testomat.io.'));
+          process.exit(1);
+        }
+
         if (client.pipeStore.runUrl) log.info( `📊 Report URL: ${pc.magenta(client.pipeStore.runUrl)}`);
 
         if (opts.kind !== 'manual') {
           log.info( `No command passed, so you need to run tests yourself:`);
           log.info( `TESTOMATIO_RUN=${runId} <command>`);
         }
+        if (opts.format) console.log(formatRunOutput(client.pipeStore, opts.format));
       } else {
         log.info( '⚠️  No API key provided. Cannot create run without TESTOMATIO key.');
         process.exit(1);
@@ -288,7 +302,11 @@ program
     }
 
     if (apiKey) {
-      await client.createRun(createRunParams).then(runTests);
+      await client.createRun(createRunParams);
+      // the runner inherits stdout, so the run data is printed first, on its own line
+      const createdOutput = formatRunOutput(client.pipeStore, opts.format);
+      if (opts.format && createdOutput) console.log(createdOutput);
+      await runTests();
     } else {
       await runTests();
     }

package/src/client.js

@@ -187,7 +187,7 @@ class Client {
       }
 
     } catch (err) {
-      console.error(APP_PREFIX, 'Error in uploadStepArtifacts for testRid', testRid, ':', err);
+      log.error('Error in uploadStepArtifacts for testRid', testRid, ':', err.message || err);
       throw err;
     }
   }
@@ -225,7 +225,7 @@ class Client {
     try {
       await this.uploadStepArtifacts(steps, rid);
     } catch (err) {
-      console.log(APP_PREFIX, 'Failed to upload step artifacts:', err);
+      log.error('Failed to upload step artifacts:', err.message || err);
     }
 
     const uploadedFiles = [];
@@ -423,7 +423,7 @@ class Client {
           const pathPadding = Math.max(...failedUploads.map(upload => upload.relativePath.length)) + 1;
 
           failedUploads.forEach(upload => {
-            console.log(
+            log.info(
               `  ${pc.gray('|')} 🔴 ${upload.relativePath.padEnd(pathPadding)} ${pc.gray(
                 `| ${upload.sizePretty.padStart(filesizeStrMaxLength)} |`,
               )}`,
@@ -439,7 +439,7 @@ class Client {
           }));
           const pathPadding = Math.max(...skippedUploads.map(upload => upload.relativePath.length)) + 1;
           skippedUploads.forEach(upload => {
-            console.log(
+            log.info(
               `  ${pc.gray('|')} 🟡 ${upload.relativePath.padEnd(pathPadding)} ${pc.gray(
                 `| ${upload.sizePretty.padStart(filesizeStrMaxLength)} |`,
               )}`,

package/src/pipe/csv.js

@@ -6,6 +6,7 @@ import pc from 'picocolors';
 import merge from 'lodash.merge';
 import { isSameTest, getCurrentDateTime, ansiRegExp } from '../utils/utils.js';
 import { CSV_HEADERS } from '../constants.js';
+import { log } from '../utils/log.js';
 
 const debug = createDebugMessages('@testomatio/reporter:pipe:csv');
 /**
@@ -76,11 +77,11 @@ class CsvPipe {
     this.checkExportDir();
 
     if (!this.outputFile) {
-      console.log(pc.yellow(`⚠️  CSV file is not set, ignoring`));
+      log.warn(pc.yellow(`⚠️  CSV file is not set, ignoring`));
       return;
     }
 
-    console.log(pc.yellow(`⏳ The test results will be added to the csv. It will take some time...`));
+    log.info(pc.yellow(`⏳ The test results will be added to the csv. It will take some time...`));
 
     try {
       // Create csv writer object
@@ -91,7 +92,7 @@ class CsvPipe {
       // Save csv file based on the current data
       return await writer.writeRecords(data);
     } catch (e) {
-      console.log('Unknown csv error: ', e);
+      log.error('Unknown csv error: ', e);
     }
   }
 
@@ -135,7 +136,7 @@ class CsvPipe {
     // Save results based on the default headers
     if (this.isEnabled) {
       await this.saveToCsv(this.results, CSV_HEADERS);
-      console.log(pc.green(`🗃️  Recording completed! You can check the result in file = ${this.outputFile}`));
+      log.info(pc.green(`🗃️  Recording completed! You can check the result in file = ${this.outputFile}`));
     }
   }
 

package/src/pipe/html.js

@@ -8,6 +8,7 @@ import { marked } from 'marked';
 import fileUrl from 'file-url';
 import { fileSystem, isSameTest, ansiRegExp, formatStep, transformEnvVarToBoolean } from '../utils/utils.js';
 import { HTML_REPORT } from '../constants.js';
+import { log } from '../utils/log.js';
 import { fileURLToPath } from 'node:url';
 
 const debug = createDebugMessages('@testomatio/reporter:pipe:html');
@@ -144,14 +145,14 @@ class HtmlPipe {
     debug('HTML tests data:', tests);
 
     if (!outputPath) {
-      console.log(pc.yellow(`🚨 HTML export path is not set, ignoring...`));
+      log.warn(pc.yellow(`🚨 HTML export path is not set, ignoring...`));
       return;
     }
 
-    console.log(pc.yellow(`⏳ The test results will be added to the HTML report. It will take some time...`));
+    log.info(pc.yellow(`⏳ The test results will be added to the HTML report. It will take some time...`));
 
     if (msg) {
-      console.log(pc.blue(msg));
+      log.info(pc.blue(msg));
     }
 
     const aggregatedTests = aggregateTestRetries(tests);
@@ -295,9 +296,9 @@ class HtmlPipe {
 
       debug('HTML tests data:', fileUrlPath);
 
-      console.log(pc.green(`📊 The HTML report was successfully generated. Full filepath: ${fileUrlPath}`));
+      log.info(pc.green(`📊 The HTML report was successfully generated. Full filepath: ${fileUrlPath}`));
     } else {
-      console.log(pc.red(`🚨 Failed to generate the HTML report.`));
+      log.error(pc.red(`🚨 Failed to generate the HTML report.`));
     }
   }
 
@@ -309,7 +310,7 @@ class HtmlPipe {
    */
   #generateHTMLReport(data, templatePath = '') {
     if (!templatePath) {
-      console.log(pc.red(`🚨 HTML template not found. Report generation is impossible!`));
+      log.error(pc.red(`🚨 HTML template not found. Report generation is impossible!`));
       return;
     }
 
@@ -320,8 +321,8 @@ class HtmlPipe {
 
       return template(data);
     } catch (e) {
-      console.log(pc.red('❌ Oops! An unknown error occurred when generating an HTML report'));
-      console.log(pc.red(e));
+      log.error(pc.red('❌ Oops! An unknown error occurred when generating an HTML report'));
+      log.error(pc.red(e));
     }
   }
 

package/src/pipe/testomatio.js

@@ -18,6 +18,7 @@ import {
   getGitCommitSha,
 } from '../utils/utils.js';
 import { parseFilterParams, generateFilterRequestParams, setS3Credentials } from '../utils/pipe_utils.js';
+import { hideTestomatioToken } from '../utils/hide_token.js';
 import { config } from '../config.js';
 import { log } from '../utils/log.js';
 
@@ -380,16 +381,15 @@ class TestomatioPipe {
       process.env.runId = this.runId;
       debug('Run created', this.runId);
     } catch (err) {
-      if (!this.apiKey) console.error('Testomat.io API key is not set');
+      if (!this.apiKey) log.error('Testomat.io API key is not set');
       const errorText = err.response?.data?.message || err.message;
       debug('Error creating run', err);
-      console.log(APP_PREFIX, errorText || err);
+      log.error(errorText || err);
       if (err.response?.status === 403) this.#disablePipe();
 
       this.#logFailedResponse(err);
 
-      console.error(
-        APP_PREFIX,
+      log.error(
         'Error creating Testomat.io report (see details above), please check if your API key is valid. Skipping report',
       );
       printCreateIssue();
@@ -409,7 +409,7 @@ class TestomatioPipe {
       this.reportingCanceledDueToReqFailures = true;
       let errorMessage = `⚠️ ${process.env.TESTOMATIO_MAX_REQUEST_FAILURES}`;
       errorMessage += ' requests were failed, reporting to Testomat aborted.';
-      console.warn(`${APP_PREFIX} ${pc.yellow(errorMessage)}`);
+      log.warn(pc.yellow(errorMessage));
     }
     return cancelReporting;
   }
@@ -557,7 +557,7 @@ class TestomatioPipe {
       const errorMessage = pc.red(
         `⚠️ Due to request failures, ${this.notReportedTestsCount} test(s) were not reported to Testomat.io`,
       );
-      console.warn(`${APP_PREFIX} ${errorMessage}`);
+      log.warn(errorMessage);
     }
 
     const { status } = params;
@@ -621,7 +621,7 @@ class TestomatioPipe {
         );
       }
     } catch (err) {
-      console.log(APP_PREFIX, 'Error updating status, skipping...', err);
+      log.error('Error updating status, skipping...', err.message || err);
       this.#logFailedResponse(err);
       printCreateIssue();
     }
@@ -665,18 +665,32 @@ class TestomatioPipe {
 
     message += `\t${pc.bold('response: ')}${pc.gray(responseBody)}\n`;
 
-    const requestBody = hideTestomatioToken(stringify(error.response?.config?.data));
+    let requestBody = hideTestomatioToken(stringify(error.response?.config?.data));
+    let requestTruncated = false;
     if (process.env.DEBUG || process.env.TESTOMATIO_DEBUG || requestBody.length < 1000) {
       // full body
       message += `\t${pc.bold('request: ')}${pc.gray(requestBody)}\n`;
     } else {
       // cut body
-      const requestBodyCut = requestBody.slice(0, 1000);
-      message += `\t${pc.bold('request: ')}${pc.gray(`${requestBodyCut}...`)}\n`;
+      requestTruncated = true;
+      requestBody = `${requestBody.slice(0, 1000)}...`;
+      message += `\t${pc.bold('request: ')}${pc.gray(requestBody)}\n`;
       message += '\trequest body is cut, run with TESTOMATIO_DEBUG=1 to see full body\n';
     }
 
-    console.log(message);
+    // the JSON line is built from the same values as the text message, with the token already hidden
+    const fields = {
+      status: statusCode,
+      method,
+      url,
+      error: apiMessage || statusText || undefined,
+      response: parseIfJson(responseBody),
+      request: parseIfJson(requestBody),
+    };
+    // a cut body is no longer valid JSON, so consumers are told why `request` is a string
+    if (requestTruncated) fields.requestTruncated = true;
+
+    log.errorWithFields(fields, message);
 
     if (error.response?.data?.message?.includes('could not be matched')) {
       this.hasUnmatchedTests = true;
@@ -693,8 +707,7 @@ function printCreateIssue() {
   if (registeredErrorHints) return;
   registeredErrorHints = true;
   process.on('exit', () => {
-    console.log(
-      APP_PREFIX,
+    log.error(
       'There was an error reporting to Testomat.io.\n',
       pc.yellow(
         'If you think this is a bug please create an issue: https://github.com/testomatio/reporter/issues/new.',
@@ -704,18 +717,6 @@ function printCreateIssue() {
   });
 }
 
-/**
- * Removes Testomatio token from string data
- *
- * @param {string} data
- * @returns {string}
- */
-function hideTestomatioToken(data) {
-  return (typeof data === 'string' ? data : '')
-    .replace(/"api_key"\s*:\s*"[^"]+"/g, '"api_key": "<hidden>"')
-    .replace(/"(tstmt_[^"]+)"/g, '"tstmt_***"');
-}
-
 /**
  * Stringifies provided data
  *
@@ -727,4 +728,18 @@ function stringify(anything, opts = { pretty: false }) {
   return typeof anything === 'string' ? anything : JSON.stringify(anything, null, opts.pretty ? 2 : undefined);
 }
 
+/**
+ * Turn a JSON string back into an object for structured logs; keeps the string if it is not JSON.
+ *
+ * @param {string} data
+ * @returns {any}
+ */
+function parseIfJson(data) {
+  try {
+    return JSON.parse(data);
+  } catch {
+    return data;
+  }
+}
+
 export default TestomatioPipe;

package/src/utils/hide_token.js

@@ -0,0 +1,12 @@
+/**
+ * Hides the Testomat.io API token in any data which is about to be printed or logged.
+ * Applied at the logger level, so a raw error object with a request body can't leak the token.
+ *
+ * @param {string} data
+ * @returns {string} The data with every token replaced, empty string if data is not a string.
+ */
+export function hideTestomatioToken(data) {
+  if (typeof data !== 'string') return '';
+
+  return data.replace(/"api_key"\s*:\s*"[^"]+"/g, '"api_key": "<hidden>"').replace(/tstmt_[\w-]+/g, 'tstmt_***');
+}

package/src/utils/log.js

@@ -1,4 +1,6 @@
+import { format as formatArgs, stripVTControlCharacters } from 'util';
 import { APP_PREFIX } from '../constants.js';
+import { hideTestomatioToken } from './hide_token.js';
 
 /**
  * Log levels for the Testomat.io reporter.
@@ -35,16 +37,54 @@ export function shouldLog(messageLevel) {
   return messageLevel <= getLogLevel() || !!process.env.TESTOMATIO_DEBUG;
 }
 
+/**
+ * Check if logs should be printed as JSON lines instead of [TESTOMATIO] prefixed text.
+ * Enabled by the CLI for `--format json` so the whole output is machine-readable.
+ * @returns {boolean}
+ */
+export function isJsonOutput() {
+  return process.env.TESTOMATIO_LOG_JSON === '1';
+}
+
+/**
+ * Render a log message as a single JSON line, e.g. `{"level":"error","message":"..."}`.
+ * Colors are stripped, so the message stays readable after parsing.
+ * @param {string} level - Log level name
+ * @param {any[]} args - Arguments as passed to the log function
+ * @param {Object} [fields] - Extra fields to add to the JSON object
+ * @returns {string}
+ */
+function jsonLine(level, args, fields = {}) {
+  const message = stripVTControlCharacters(formatArgs(...args)).trim();
+  return hideTestomatioToken(JSON.stringify({ ...fields, level, message }));
+}
+
+/**
+ * Render the arguments of a log function as text, with the API token hidden.
+ * Errors and other objects are formatted the way console does it.
+ * @param {any[]} args - Arguments as passed to the log function
+ * @returns {string}
+ */
+function textLine(args) {
+  return hideTestomatioToken(formatArgs(...args));
+}
+
 /**
  * Log an info message with [TESTOMATIO] prefix.
  * Only logs when TESTOMATIO_LOG_LEVEL is INFO.
  * @param {...any} args - Arguments to log
  */
 export function info(...args) {
-  if (shouldLog(LOG_LEVELS.INFO)) {
-    const fn = process.env.TESTOMATIO_LOG_STDERR === '1' ? console.error : console.log;
-    fn(APP_PREFIX, ...args);
+  if (!shouldLog(LOG_LEVELS.INFO)) return;
+
+  let fn = console.log;
+  if (process.env.TESTOMATIO_LOG_STDERR === '1') fn = console.error;
+
+  if (isJsonOutput()) {
+    fn(jsonLine('info', args));
+    return;
   }
+  fn(APP_PREFIX, textLine(args));
 }
 
 /**
@@ -53,9 +93,13 @@ export function info(...args) {
  * @param {...any} args - Arguments to log
  */
 export function warn(...args) {
-  if (shouldLog(LOG_LEVELS.WARN)) {
-    console.warn(APP_PREFIX, ...args);
+  if (!shouldLog(LOG_LEVELS.WARN)) return;
+
+  if (isJsonOutput()) {
+    console.warn(jsonLine('warn', args));
+    return;
   }
+  console.warn(APP_PREFIX, textLine(args));
 }
 
 /**
@@ -64,9 +108,29 @@ export function warn(...args) {
  * @param {...any} args - Arguments to log
  */
 export function error(...args) {
-  if (shouldLog(LOG_LEVELS.ERROR)) {
-    console.error(APP_PREFIX, ...args);
+  if (!shouldLog(LOG_LEVELS.ERROR)) return;
+
+  if (isJsonOutput()) {
+    console.error(jsonLine('error', args));
+    return;
+  }
+  console.error(APP_PREFIX, textLine(args));
+}
+
+/**
+ * Log an error which carries structured data, e.g. a failed API request.
+ * The fields are added to the JSON line; in text mode only the message is printed.
+ * @param {Object} fields - Extra fields to add to the JSON object
+ * @param {...any} args - Arguments to log as text
+ */
+export function errorWithFields(fields, ...args) {
+  if (!shouldLog(LOG_LEVELS.ERROR)) return;
+
+  if (isJsonOutput()) {
+    console.error(jsonLine('error', args, fields));
+    return;
   }
+  console.error(APP_PREFIX, textLine(args));
 }
 
 /**
@@ -82,6 +146,8 @@ export const log = {
   info,
   warn,
   error,
+  errorWithFields,
+  isJsonOutput,
   getLogLevel,
   shouldLog,
   LOG_LEVELS,

package/src/utils/pipe_utils.js

@@ -304,6 +304,27 @@ function plannedTestsLabel(tests, testsCount) {
   return `**${knownTestsCount}** tests and **${suitesCount}** suites planned`;
 }
 
+/**
+ * Format the created run for machine-readable output of `start` and `run`.
+ * `json` prints an object with the run details, any other format prints the bare run id.
+ *
+ * @param {{runId?: string, runUrl?: string, runPublicUrl?: string}} store - Pipe store of the client.
+ * @param {string} [format] - Value of the CLI `--format` option.
+ * @returns {string} Empty string if there is no run id.
+ */
+function formatRunOutput(store, format) {
+  const runId = store?.runId;
+  if (!runId) return '';
+
+  if (format !== 'json') return runId;
+
+  const output = { runId };
+  if (store.runUrl) output.runUrl = store.runUrl;
+  if (store.runPublicUrl) output.runPublicUrl = store.runPublicUrl;
+
+  return JSON.stringify(output);
+}
+
 export {
   updateFilterType,
   parseFilterParams,
@@ -317,6 +338,7 @@ export {
   plannedTestsLabel,
   parsePipeOptions,
   formatFilterListIds,
+  formatRunOutput,
   getObjectSize,
   splitTestsIntoChunks,
 };