testomatio-reporter-cli 2.19.0 → 2.21.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testomatio-reporter-cli",
-  "version": "2.19.0",
+  "version": "2.21.0",
   "description": "Yarn Berry compatible standalone CLI for @testomatio/reporter",
   "type": "module",
   "bin": {

package/src/adapter/codecept.js

@@ -11,6 +11,8 @@ import codeceptjs from 'codeceptjs';
 import { log } from '../utils/log.js';
 
 const debug = createDebugMessages('@testomatio/reporter:adapter:codeceptjs');
+
+global.TESTOMATIO_ADAPTER = 'codeceptjs';
 // @ts-ignore
 if (!global.codeceptjs) {
   // @ts-ignore
@@ -18,7 +20,7 @@ if (!global.codeceptjs) {
 }
 
 // @ts-ignore
-const { event, recorder, codecept, output } = global.codeceptjs;
+const { event, recorder, codecept, output, container } = global.codeceptjs;
 
 const [, MAJOR_VERSION, MINOR_VERSION] = codecept
   .version()
@@ -57,6 +59,7 @@ function CodeceptReporter(config) {
   let isRunFinalized = false;
 
   const testTimeMap = {};
+  const failedUrls = new Map();
   const clientConfig = buildCodeceptClientConfig(config);
   const client = new TestomatClient(clientConfig);
 
@@ -241,6 +244,22 @@ function CodeceptReporter(config) {
     }, true);
   });
 
+  event.dispatcher.on(event.test.failed, (test, _err, hookName) => {
+    // no browser in suite hooks
+    if (hookName === 'BeforeSuite' || hookName === 'AfterSuite') return;
+    const helper = Object.values(container.helpers()).find(h => typeof h.grabCurrentUrl === 'function');
+    if (!helper) return;
+
+    recorder.add('url of failed test', async () => {
+      try {
+        const url = await helper.grabCurrentUrl();
+        if (url) failedUrls.set(test.uid, url);
+      } catch (err) {
+        debug('Failed to grab url of failed test', err?.message || err);
+      }
+    }, true);
+  });
+
   event.dispatcher.on(event.test.skipped, test => {
     const { uid, tags, title } = test.simplify();
 
@@ -272,6 +291,9 @@ function CodeceptReporter(config) {
     const keyValues = services.keyValues.get(test.fullTitle());
     const links = services.links.get(test.fullTitle());
     const screenshotOnFailPath = artifacts.screenshot || null;
+    const meta = { ...keyValues, ...test.meta };
+    if (test.state === STATUS.FAILED && failedUrls.has(uid)) meta.failedUrl = failedUrls.get(uid);
+    failedUrls.delete(uid);
 
     // Build step hierarchy with screenshot from screenshotOnFail
     const stepHierarchy = buildUnifiedStepHierarchy(
@@ -295,7 +317,7 @@ function CodeceptReporter(config) {
       logs,
       links,
       manuallyAttachedArtifacts,
-      meta: { ...keyValues, ...test.meta },
+      meta,
     });
     reportTestPromises.push(reportTestPromise);
 

package/src/adapter/cucumber/current.js

@@ -8,6 +8,8 @@ import { getTestomatIdFromTestTitle, fileSystem } from '../../utils/utils.js';
 import { config } from '../../config.js';
 import { services } from '../../services/index.js';
 
+global.TESTOMATIO_ADAPTER = 'cucumber';
+
 const { GherkinDocumentParser, PickleParser } = formatterHelpers;
 const { getGherkinScenarioLocationMap, getGherkinStepMap } = GherkinDocumentParser;
 const { getPickleStepMap } = PickleParser;

package/src/adapter/cypress-plugin/index.js

@@ -3,6 +3,8 @@ import { getTestomatIdFromTestTitle, parseSuite } from '../../utils/utils.js';
 import TestomatClient from '../../client.js';
 import { config } from '../../config.js';
 
+global.TESTOMATIO_ADAPTER = 'cypress';
+
 const testomatioReporter = on => {
   if (!config.TESTOMATIO) {
     console.log('TESTOMATIO key is empty, ignoring reports');

package/src/adapter/e2e.js

@@ -0,0 +1,248 @@
+import path from 'path';
+import createDebugMessages from 'debug';
+import TestomatioClient from '../client.js';
+import { STATUS, SCREENSHOTS_ON_STEPS } from '../constants.js';
+import { getTestomatIdFromTestTitle, truncate } from '../utils/utils.js';
+import { formatStep, addStatusToStep, addArtifactsToStep } from './utils/step-formatter.js';
+import { log } from '../utils/log.js';
+
+const debug = createDebugMessages('@testomatio/reporter:adapter-e2e');
+
+/**
+ * Reporter for the e2e framework (https://github.com/tester-army/e2e).
+ * Add it to `reporters` in `e2e.config.ts`: `reporters: ['list', testomatio()]`.
+ *
+ * The config is loaded by every worker, so the client is created only inside
+ * handlers, which run in the runner process.
+ *
+ * @param {{ apiKey?: string }} [options]
+ */
+export function testomatio(options = {}) {
+  let client = null;
+  let runPromise = null;
+
+  function getClient() {
+    if (!client) client = new TestomatioClient({ apiKey: options.apiKey });
+    return client;
+  }
+
+  function startRun() {
+    if (!runPromise) {
+      runPromise = getClient()
+        .createRun()
+        .catch(err => log.error('Failed to create Testomat.io run:', err?.message || err));
+    }
+    return runPromise;
+  }
+
+  return {
+    name: 'testomatio',
+
+    onEvent(event) {
+      try {
+        if (event?.type === 'run-started') startRun();
+      } catch (err) {
+        debug('onEvent failed', err);
+      }
+    },
+
+    async onRunFinished(run, signal) {
+      const reporter = getClient();
+      await startRun();
+
+      const { report, artifactsRoot } = run;
+      const groups = new Map((report.run.serialGroups || []).map(group => [group.id, group]));
+
+      const promises = [];
+      for (const result of report.run.results) {
+        if (signal?.aborted) break;
+        // results left out by `--last-failed` belong to the previous run
+        if (result.selected === false) continue;
+        const { status, data } = toTestData(result, groups, artifactsRoot);
+        promises.push(reporter.addTestRun(status, data));
+      }
+      await Promise.all(promises);
+
+      const duration = (Date.parse(report.run.finishedAt) - Date.parse(report.run.startedAt)) / 1000;
+      await reporter.updateRunStatus(runStatus(report.run.status), { duration });
+
+      const runUrl = reporter.pipeStore?.runUrl;
+      if (runUrl) return [{ label: 'Testomat.io', text: runUrl }];
+    },
+  };
+}
+
+/**
+ * Converts one report-1 result into Testomat.io test data.
+ * Only the final attempt is reported; earlier attempts are counted as retries.
+ */
+export function toTestData(result, groups, artifactsRoot) {
+  const attempts = attemptsOf(result, groups);
+  const final = attempts.at(-1);
+  const titlePath = [...result.titlePath];
+  const title = titlePath.pop();
+
+  let suiteTitle = titlePath.join(' › ');
+  if (!suiteTitle) suiteTitle = path.basename(result.file);
+
+  const meta = {
+    target: result.targetId,
+    platform: result.platform,
+    browser: final?.environment?.browser,
+  };
+  if (result.agent && result.agent !== 'default') meta.agent = result.agent;
+  if (result.repeat) meta.repeat = result.repeat;
+  if (attempts.length > 1) meta.retryCount = attempts.length - 1;
+  if (result.status === 'flaky') meta.isFlaky = true;
+  const status = testStatus(result.status);
+  if (status === STATUS.FAILED && final?.failure?.url) meta.failedUrl = final.failure.url;
+
+  const filesById = new Map();
+  for (const artifact of final?.artifacts || []) {
+    if (artifact.path) {
+      filesById.set(artifact.id, {
+        path: path.join(artifactsRoot, artifact.path),
+        type: artifact.mediaType,
+        title: artifact.kind,
+      });
+    } else if (artifact.url) {
+      // hosted engines keep videos remotely
+      meta[artifact.kind] = artifact.url;
+    }
+  }
+  const files = [...filesById.values()];
+
+  const steps = (final?.steps || []).map(step => toStep(step, filesById));
+
+  const data = {
+    rid: result.id,
+    title,
+    suite_title: suiteTitle,
+    file: result.file,
+    test_id: getTestomatIdFromTestTitle(title),
+    tags: result.tags || [],
+    time: final?.durationMs || 0,
+    timestamp: timestampOf(final),
+    error: toError(final?.error, final?.secondaryErrors),
+    steps: steps.length ? steps : undefined,
+    logs: logsOf(result, final),
+    files: files.length ? files : undefined,
+    meta,
+  };
+
+  return { status, data };
+}
+
+/** Attempts of a result; a serial member keeps its part of the group's attempts. */
+function attemptsOf(result, groups) {
+  if (!result.serialGroupId) return result.attempts || [];
+  const groupAttempts = groups.get(result.serialGroupId)?.attempts || [];
+  return groupAttempts.map(attempt => {
+    const member = attempt.members?.find(candidate => candidate.testId === result.testId);
+    return {
+      status: member?.status || attempt.status,
+      error: member?.error || attempt.error,
+      failure: member?.failure || attempt.failure,
+      steps: member?.steps || [],
+      appLog: member?.appLog || [],
+      skip: member?.skip,
+      durationMs: member?.durationMs,
+      startedAt: member?.startedAt || attempt.startedAt,
+      artifacts: attempt.artifacts || [],
+      secondaryErrors: [...(member?.secondaryErrors || []), ...(attempt.secondaryErrors || [])],
+      environment: attempt.environment,
+    };
+  });
+}
+
+function timestampOf(attempt) {
+  if (!attempt?.startedAt) return undefined;
+  return Date.parse(attempt.startedAt);
+}
+
+function testStatus(status) {
+  if (status === 'passed' || status === 'flaky') return STATUS.PASSED;
+  if (status === 'skipped') return STATUS.SKIPPED;
+  // failed, timed-out, interrupted: the test reached no passing verdict
+  return STATUS.FAILED;
+}
+
+function runStatus(status) {
+  if (status === 'passed') return STATUS.PASSED;
+  return STATUS.FAILED;
+}
+
+function toError(error, secondaryErrors = []) {
+  if (!error) return undefined;
+  /** @type {Error & { expected?: string, actual?: string }} */
+  const err = new Error(error.message);
+  err.name = error.code;
+  err.stack = `${error.code}: ${error.message}\n${error.stack || sourceFrame(error.source)}`;
+  if (error.details?.expected && error.details?.observed) {
+    err.expected = error.details.expected;
+    err.actual = error.details.observed;
+  }
+  for (const secondary of secondaryErrors) {
+    err.stack += `\n\nAlso failed (${secondary.phase || secondary.category}) ${secondary.code}: ${secondary.message}`;
+  }
+  return err;
+}
+
+function sourceFrame(source) {
+  if (!source) return '';
+  return `    at ${source.file}:${source.line}:${source.column}`;
+}
+
+function toStep(step, filesById) {
+  let category = 'user';
+  if (step.phase === 'beforeEach' || step.phase === 'afterEach') category = 'hook';
+
+  let title = step.api;
+  if (step.label) title = `${step.api} ${step.label}`;
+
+  const formatted = formatStep({
+    category,
+    title,
+    duration: step.durationMs,
+    log: stepLog(step),
+  });
+
+  let status = 'failed';
+  if (step.status === 'passed') status = 'passed';
+  addStatusToStep(formatted, status, step.error);
+
+  if (step.error) {
+    formatted.error = { message: truncate(`${step.error.code}: ${step.error.message}`, 250) };
+  }
+
+  if (SCREENSHOTS_ON_STEPS) {
+    const screenshot = (step.artifacts || []).map(id => filesById.get(id)).find(file => file?.type === 'image/png');
+    if (screenshot) addArtifactsToStep(formatted, { screenshot: screenshot.path });
+  }
+
+  return formatted;
+}
+
+/** Agent reasoning and non-binary step statuses, which a passed/failed step would lose. */
+function stepLog(step) {
+  const lines = [];
+  if (step.status !== 'passed' && step.status !== 'failed') lines.push(`status: ${step.status}`);
+  if (step.explanation) lines.push(step.explanation);
+  for (const event of step.events || []) {
+    if (event.reasoning) lines.push(event.reasoning);
+  }
+  return lines.join('\n');
+}
+
+function logsOf(result, final) {
+  const lines = [];
+  const skip = final?.skip || result.skip;
+  if (skip) lines.push(`Skipped (${skip.cause}): ${skip.reason}`);
+  if (final?.appLog?.length) {
+    lines.push('App log:');
+    for (const line of final.appLog) lines.push(`[${line.source}] ${line.level}: ${line.text}`);
+  }
+  return lines.join('\n');
+}
+
+export default testomatio;

package/src/adapter/jasmine.js

@@ -2,6 +2,8 @@ import TestomatClient from '../client.js';
 import { getTestomatIdFromTestTitle, ansiRegExp } from '../utils/utils.js';
 import { STATUS } from '../constants.js';
 
+global.TESTOMATIO_ADAPTER = 'jasmine';
+
 class JasmineReporter {
   constructor(options) {
     this.testTimeMap = {};

package/src/adapter/jest.js

@@ -8,6 +8,8 @@ import createDebugMessages from 'debug';
 
 const debug = createDebugMessages('@testomatio/reporter:adapter-jest');
 
+global.TESTOMATIO_ADAPTER = 'jest';
+
 export class JestReporter {
   constructor(globalConfig, options) {
     this._globalConfig = globalConfig;

package/src/adapter/mocha.js

@@ -6,6 +6,8 @@ import { config } from '../config.js';
 import { services } from '../services/index.js';
 import pc from 'picocolors';
 
+global.TESTOMATIO_ADAPTER = 'mocha';
+
 const {
   EVENT_RUN_BEGIN,
   EVENT_RUN_END,
@@ -18,157 +20,160 @@ const {
   EVENT_TEST_END,
 } = Mocha.Runner.constants;
 
-function MochaReporter(runner, opts) {
-  Mocha.reporters.Base.call(this, runner);
-  let passes = 0;
-  let failures = 0;
-  let skipped = 0;
-  // let artifactStore;
-
-  const apiKey = opts?.reporterOptions?.apiKey || config.TESTOMATIO;
-
-  const client = new TestomatClient({ apiKey });
-
-  // Track hook failures
-  const hookFailures = new Map();
-
-  runner.on(EVENT_RUN_BEGIN, () => {
-    client.createRun();
-
-    // clear dir with artifacts/logs
-    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
-  });
-
-  runner.on(EVENT_SUITE_BEGIN, async suite => {
-    services.setContext(suite.fullTitle());
-  });
-
-  runner.on(EVENT_SUITE_END, async suite => {
-    if (hookFailures.has(suite.fullTitle())) {
-      const { error, suiteTitle } = hookFailures.get(suite.fullTitle());
-
-      for (const test of suite.tests) {
-        if (test.state === 'pending' || !test.state) {
-          const testId = getTestomatIdFromTestTitle(test.title);
-          const artifacts = services.artifacts.get(test.fullTitle());
-          const keyValues = services.keyValues.get(test.fullTitle());
-          const links = services.links.get(test.fullTitle());
-
-          client.addTestRun(STATUS.FAILED, {
-            error,
-            suite_title: suiteTitle || suite.title,
-            file: suite.file,
-            test_id: testId,
-            title: test.title,
-            code: process.env.TESTOMATIO_UPDATE_CODE ? test.body.toString() : '',
-            time: 0,
-            manuallyAttachedArtifacts: artifacts,
-            meta: keyValues,
-            links,
-          });
+// ES class so it works with mocha 12, where Base is a class and utils.inherits is gone
+class MochaReporter extends Mocha.reporters.Base {
+  constructor(runner, opts) {
+    super(runner, opts);
+    let passes = 0;
+    let failures = 0;
+    let skipped = 0;
+    // let artifactStore;
+
+    const apiKey = opts?.reporterOptions?.apiKey || config.TESTOMATIO;
+
+    const client = new TestomatClient({ apiKey });
+
+    // Track hook failures
+    const hookFailures = new Map();
+
+    runner.on(EVENT_RUN_BEGIN, () => {
+      client.createRun();
+
+      // clear dir with artifacts/logs
+      fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
+    });
+
+    runner.on(EVENT_SUITE_BEGIN, async suite => {
+      services.setContext(suite.fullTitle());
+    });
+
+    runner.on(EVENT_SUITE_END, async suite => {
+      if (hookFailures.has(suite.fullTitle())) {
+        const { error, suiteTitle } = hookFailures.get(suite.fullTitle());
+
+        for (const test of suite.tests) {
+          if (test.state === 'pending' || !test.state) {
+            const testId = getTestomatIdFromTestTitle(test.title);
+            const artifacts = services.artifacts.get(test.fullTitle());
+            const keyValues = services.keyValues.get(test.fullTitle());
+            const links = services.links.get(test.fullTitle());
+
+            client.addTestRun(STATUS.FAILED, {
+              error,
+              suite_title: suiteTitle || suite.title,
+              file: suite.file,
+              test_id: testId,
+              title: test.title,
+              code: process.env.TESTOMATIO_UPDATE_CODE ? test.body.toString() : '',
+              time: 0,
+              manuallyAttachedArtifacts: artifacts,
+              meta: keyValues,
+              links,
+            });
+          }
         }
       }
-    }
-
-    services.setContext(null);
-  });
-
-  runner.on(EVENT_TEST_BEGIN, async test => {
-    services.setContext(test.fullTitle());
-  });
-
-  runner.on(EVENT_TEST_END, async () => {
-    services.setContext(null);
-  });
-
-  runner.on(EVENT_TEST_PASS, async test => {
-    passes += 1;
-
-    console.log(pc.bold(pc.green('✔')), test.fullTitle());
-    const testId = getTestomatIdFromTestTitle(test.title);
-
-    const logs = getTestLogs(test);
-    const artifacts = services.artifacts.get(test.fullTitle());
-    const keyValues = services.keyValues.get(test.fullTitle());
-    const links = services.links.get(test.fullTitle());
-
-    client.addTestRun(STATUS.PASSED, {
-      test_id: testId,
-      suite_title: getSuiteTitle(test),
-      title: getTestName(test),
-      code: process.env.TESTOMATIO_UPDATE_CODE ? test.body.toString() : '',
-      file: getFile(test),
-      time: test.duration,
-      logs,
-      manuallyAttachedArtifacts: artifacts,
-      meta: keyValues,
-      links,
+
+      services.setContext(null);
+    });
+
+    runner.on(EVENT_TEST_BEGIN, async test => {
+      services.setContext(test.fullTitle());
+    });
+
+    runner.on(EVENT_TEST_END, async () => {
+      services.setContext(null);
     });
-  });
-
-  runner.on(EVENT_TEST_PENDING, test => {
-    skipped += 1;
-    console.log('skip: %s', test.fullTitle());
-    const testId = getTestomatIdFromTestTitle(test.title);
-    const artifacts = services.artifacts.get(test.fullTitle());
-    const keyValues = services.keyValues.get(test.fullTitle());
-    const links = services.links.get(test.fullTitle());
-
-    client.addTestRun(STATUS.SKIPPED, {
-      title: getTestName(test),
-      suite_title: getSuiteTitle(test),
-      code: process.env.TESTOMATIO_UPDATE_CODE ? test.body.toString() : '',
-      file: getFile(test),
-      test_id: testId,
-      time: test.duration,
-      manuallyAttachedArtifacts: artifacts,
-      meta: keyValues,
-      links,
+
+    runner.on(EVENT_TEST_PASS, async test => {
+      passes += 1;
+
+      console.log(pc.bold(pc.green('✔')), test.fullTitle());
+      const testId = getTestomatIdFromTestTitle(test.title);
+
+      const logs = getTestLogs(test);
+      const artifacts = services.artifacts.get(test.fullTitle());
+      const keyValues = services.keyValues.get(test.fullTitle());
+      const links = services.links.get(test.fullTitle());
+
+      client.addTestRun(STATUS.PASSED, {
+        test_id: testId,
+        suite_title: getSuiteTitle(test),
+        title: getTestName(test),
+        code: process.env.TESTOMATIO_UPDATE_CODE ? test.body.toString() : '',
+        file: getFile(test),
+        time: test.duration,
+        logs,
+        manuallyAttachedArtifacts: artifacts,
+        meta: keyValues,
+        links,
+      });
     });
-  });
 
-  runner.on(EVENT_TEST_FAIL, async (test, err) => {
-    failures += 1;
-    console.log(pc.bold(pc.red('✖')), test.fullTitle(), pc.gray(err.message));
+    runner.on(EVENT_TEST_PENDING, test => {
+      skipped += 1;
+      console.log('skip: %s', test.fullTitle());
+      const testId = getTestomatIdFromTestTitle(test.title);
+      const artifacts = services.artifacts.get(test.fullTitle());
+      const keyValues = services.keyValues.get(test.fullTitle());
+      const links = services.links.get(test.fullTitle());
+
+      client.addTestRun(STATUS.SKIPPED, {
+        title: getTestName(test),
+        suite_title: getSuiteTitle(test),
+        code: process.env.TESTOMATIO_UPDATE_CODE ? test.body.toString() : '',
+        file: getFile(test),
+        test_id: testId,
+        time: test.duration,
+        manuallyAttachedArtifacts: artifacts,
+        meta: keyValues,
+        links,
+      });
+    });
+
+    runner.on(EVENT_TEST_FAIL, async (test, err) => {
+      failures += 1;
+      console.log(pc.bold(pc.red('✖')), test.fullTitle(), pc.gray(err.message));
+
+      const isHookFailure = test.title.includes('before each') || test.title.includes('after each');
 
-    const isHookFailure = test.title.includes('before each') || test.title.includes('after each');
+      if (isHookFailure && test.parent) {
+        hookFailures.set(test.parent.fullTitle(), {
+          error: err,
+          suiteTitle: getSuiteTitle(test),
+        });
+        return;
+      }
+
+      const testId = getTestomatIdFromTestTitle(test.title);
 
-    if (isHookFailure && test.parent) {
-      hookFailures.set(test.parent.fullTitle(), {
+      const logs = getTestLogs(test);
+      const artifacts = services.artifacts.get(test.fullTitle());
+      const keyValues = services.keyValues.get(test.fullTitle());
+      const links = services.links.get(test.fullTitle());
+
+      client.addTestRun(STATUS.FAILED, {
         error: err,
-        suiteTitle: getSuiteTitle(test),
+        suite_title: getSuiteTitle(test),
+        file: getFile(test),
+        test_id: testId,
+        title: getTestName(test),
+        code: process.env.TESTOMATIO_UPDATE_CODE ? test.body.toString() : '',
+        time: test.duration,
+        logs,
+        manuallyAttachedArtifacts: artifacts,
+        meta: keyValues,
+        links,
       });
-      return;
-    }
-
-    const testId = getTestomatIdFromTestTitle(test.title);
-
-    const logs = getTestLogs(test);
-    const artifacts = services.artifacts.get(test.fullTitle());
-    const keyValues = services.keyValues.get(test.fullTitle());
-    const links = services.links.get(test.fullTitle());
-
-    client.addTestRun(STATUS.FAILED, {
-      error: err,
-      suite_title: getSuiteTitle(test),
-      file: getFile(test),
-      test_id: testId,
-      title: getTestName(test),
-      code: process.env.TESTOMATIO_UPDATE_CODE ? test.body.toString() : '',
-      time: test.duration,
-      logs,
-      manuallyAttachedArtifacts: artifacts,
-      meta: keyValues,
-      links,
     });
-  });
-
-  runner.on(EVENT_RUN_END, () => {
-    const status = failures === 0 ? STATUS.PASSED : STATUS.FAILED;
-    console.log(pc.bold(status), `${passes} passed, ${failures} failed, ${skipped} skipped`);
-    // @ts-ignore
-    client.updateRunStatus(status);
-  });
+
+    runner.on(EVENT_RUN_END, () => {
+      const status = failures === 0 ? STATUS.PASSED : STATUS.FAILED;
+      console.log(pc.bold(status), `${passes} passed, ${failures} failed, ${skipped} skipped`);
+      // @ts-ignore
+      client.updateRunStatus(status);
+    });
+  }
 }
 
 function getTestLogs(test) {
@@ -204,8 +209,4 @@ function getTestName(test) {
   return test.title;
 }
 
-// To have this reporter "extend" a built-in reporter uncomment the following line:
-// @ts-ignore
-Mocha.utils.inherits(MochaReporter, Mocha.reporters.Spec);
-
 export default MochaReporter;