testomatio-reporter-cli 2.8.4 → 2.8.5-beta.2-yarn

package/src/adapter/cucumber/current.js

@@ -0,0 +1,230 @@
+import { Formatter, formatterHelpers } from '@cucumber/cucumber';
+
+import pc from 'picocolors';
+import fs from 'fs';
+import { STATUS, TESTOMAT_TMP_STORAGE_DIR } from '../../constants.js';
+import TestomatClient from '../../client.js';
+import { getTestomatIdFromTestTitle, fileSystem } from '../../utils/utils.js';
+import { config } from '../../config.js';
+import { services } from '../../services/index.js';
+
+const { GherkinDocumentParser, PickleParser } = formatterHelpers;
+const { getGherkinScenarioLocationMap, getGherkinStepMap } = GherkinDocumentParser;
+const { getPickleStepMap } = PickleParser;
+
+function getTestId(scenario) {
+  if (scenario) {
+    for (const tag of scenario.tags) {
+      const testId = getTestomatIdFromTestTitle(tag.name);
+      if (testId) return testId;
+    }
+  }
+
+  return null;
+}
+
+class CucumberReporter extends Formatter {
+  constructor(options) {
+    super(options);
+    options.eventBroadcaster.on('envelope', this.parseEnvelope.bind(this));
+    this.failures = [];
+    this.cases = [];
+
+    this.client = new TestomatClient({ apiKey: options.apiKey || config.TESTOMATIO });
+    this.client.createRun();
+    this.status = STATUS.PASSED;
+  }
+
+  parseEnvelope(envelope) {
+    if (envelope.testRunStarted) {
+      fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
+    }
+    if (envelope.testCaseStarted && this.client) {
+      this.onTestCaseStarted(envelope.testCaseStarted);
+    }
+    if (envelope.testCaseFinished) this.onTestCaseFinished(envelope.testCaseFinished);
+    if (envelope.testRunFinished) this.onTestRunFinished(envelope);
+  }
+
+  onTestCaseStarted(testCaseStarted) {
+    const testCaseAttempt = this.eventDataCollector.getTestCaseAttempt(testCaseStarted.id);
+    if (!global.testomatioDataStore) global.testomatioDataStore = {};
+
+    const testTitle = testCaseAttempt.pickle.name + testCaseAttempt.pickle.uri;
+    services.setContext(testTitle);
+  }
+
+  onTestCaseFinished(testCaseFinished) {
+    const testCaseAttempt = this.eventDataCollector.getTestCaseAttempt(testCaseFinished.testCaseStartedId);
+
+    let example;
+
+    let status = STATUS.PASSED;
+    let color = 'green';
+    let message;
+
+    this.cases.push(testCaseAttempt);
+
+    if (testCaseAttempt.worstTestStepResult) {
+      if (testCaseAttempt.worstTestStepResult.status === 'SKIPPED') {
+        status = STATUS.SKIPPED;
+      }
+      // if (testCaseAttempt.worstTestStepResult.status === 'UNDEFINED') {
+      //   status = STATUS.SKIPPED;
+      //   message = 'Undefined steps. Implement missing steps and rerun this scenario';
+      // }
+      if (testCaseAttempt.worstTestStepResult.status === 'FAILED') {
+        message = testCaseAttempt?.worstTestStepResult?.message;
+        status = STATUS.FAILED;
+      }
+      color = getStatusColor(testCaseAttempt.worstTestStepResult.status);
+      if (status !== STATUS.PASSED) this.failures.push(testCaseAttempt);
+    }
+
+    if (testCaseAttempt.pickle.astNodeIds.length > 1) {
+      example = getExample(testCaseAttempt);
+      // @ts-ignore
+      testCaseAttempt.example = example;
+    }
+
+    const scenario = testCaseAttempt.pickle.name;
+    // this may broke something (it is supposed to work, but I am sure it did not)
+    // const testId = testCaseAttempt.pickle.id;
+    const testId = getTestId(testCaseAttempt.pickle);
+
+    let exampleString = '';
+    if (example) exampleString = ` ${example.join(' | ')}`;
+    let cliMessage = `${pc.bold(scenario)}${exampleString}: ${pc[color](pc.bold(status.toUpperCase()))} `;
+
+    if (message) cliMessage += pc.gray(message.split('\n')[0]);
+    console.log(cliMessage);
+
+    if (status !== STATUS.PASSED && status !== STATUS.SKIPPED) {
+      this.status = STATUS.FAILED;
+    }
+
+    const time = Object.values(testCaseAttempt.stepResults)
+      .map(t => t.duration)
+      .reduce((sum, duration) => sum + duration.seconds * 1000 + duration.nanos / 1000000, 0);
+
+    if (!this.client) return;
+
+    const testTitle = testCaseAttempt.pickle.name + testCaseAttempt.pickle.uri;
+    const logs = services.logger.getLogs(testTitle).join('\n');
+    const artifacts = services.artifacts.get(testTitle);
+    const keyValues = services.keyValues.get(testTitle);
+    const links = services.links.get(testTitle);
+
+    this.client.addTestRun(status, {
+      // error: testCaseAttempt.worstTestStepResult.message,
+      message,
+      steps: getSteps(testCaseAttempt)
+        .map(s => s.toString())
+        .join('\n')
+        .trim(),
+      example: { ...example },
+      logs,
+      links,
+      manuallyAttachedArtifacts: artifacts,
+      meta: keyValues,
+      title: scenario,
+      test_id: testId,
+      time,
+    });
+
+    services.setContext(null);
+  }
+
+  onTestRunFinished(envelope) {
+    if (this.failures.length > 0) {
+      console.log(pc.bold('\nSUMMARY:\n\n'));
+
+      this.failures.forEach((tc, i) => {
+        let message = `  ${i + 1}) ${tc.pickle.name}\n`;
+
+        const steps = getSteps(tc);
+
+        steps.forEach(s => {
+          message += `     ${s.toString()}\n`;
+        });
+
+        console.log(message);
+        if (tc?.worstTestStepResult?.message) {
+          console.log(pc.red(tc?.worstTestStepResult?.message));
+        }
+        console.log();
+      });
+    }
+
+    const { testRunFinished } = envelope;
+
+    const bgColor = testRunFinished.success ? 'bgGreen' : 'bgRed';
+    const prefixSummary = `${pc.bold(testRunFinished.success ? ' SUCCESS ' : ' FALIURE ')}`;
+    console.log();
+    console.log(pc[bgColor](` ${prefixSummary} | Total Scenarios: ${pc.bold(this.cases.length)} `));
+
+    if (!this.client) return;
+
+    // @ts-ignore
+    this.client.updateRunStatus(testRunFinished.success ? STATUS.PASSED : STATUS.FAILED);
+  }
+}
+
+function getSteps(tc) {
+  const stepIds = Object.keys(tc.stepResults);
+  const pickleSteps = getPickleStepMap(tc.pickle);
+  return stepIds
+    .map(stepId => {
+      const ts = tc.testCase.testSteps.find(t => t.id === stepId);
+      if (!ts) return;
+      if (!ts.pickleStepId) return;
+      const result = tc.stepResults[stepId];
+      const pickleStep = pickleSteps[ts.pickleStepId];
+      const sourceStepId = pickleStep.astNodeIds[0];
+      const step = {
+        text: pickleStep.text,
+        duration: result.duration,
+        status: result.status,
+      };
+      const color = getStatusColor(result.status);
+      if (sourceStepId && getGherkinStepMap(tc.gherkinDocument)[sourceStepId]) {
+        step.keyword = getGherkinStepMap(tc.gherkinDocument)[sourceStepId].keyword;
+      }
+      step.toString = function toString() {
+        const duration = step.duration.seconds * 1000 + step.duration.nanos / 1000000;
+        const durationString = ` ${pc.gray(`(${Number(duration).toFixed(2)}ms)`)}`;
+        const stepString = `${pc.bold(this.keyword)}${this.text}`.trim();
+        if (color === 'red') return pc.red(stepString) + durationString;
+        if (color === 'yellow') return pc.yellow(stepString) + durationString;
+        return stepString + durationString;
+      };
+      return step;
+    })
+    .filter(s => !!s);
+}
+
+function getStatusColor(status) {
+  if (status === 'UNDEFINED') return 'yellow';
+  if (status === 'SKIPPED') return 'yellow';
+  if (status === 'FAILED') return 'red';
+  return 'green';
+}
+
+function getExample(testCaseAttempt) {
+  const nodesMap = getGherkinScenarioLocationMap(testCaseAttempt.gherkinDocument);
+  const exampleNodeId = testCaseAttempt.pickle.astNodeIds[1];
+  if (!nodesMap[exampleNodeId]) return;
+  const featureDoc = fs.readFileSync(testCaseAttempt.gherkinDocument.uri).toString();
+  const { line } = nodesMap[exampleNodeId];
+  const example = featureDoc.split('\n')[line - 1];
+  if (example) {
+    return example
+      .trim()
+      .split('|')
+      .filter(r => !!r)
+      .map(r => r.trim());
+  }
+}
+
+export { CucumberReporter };
+export default CucumberReporter;

package/src/adapter/cucumber/legacy.js

@@ -0,0 +1,158 @@
+// import { Formatter } from 'cucumber';
+
+// import pc from 'picocolors';
+// import { getTestomatIdFromTestTitle, fileSystem } from '../../utils/utils.js';
+// import { STATUS, TESTOMAT_TMP_STORAGE_DIR } from '../../constants.js';
+// import TestomatClient from '../../client.js';
+// import {config} from '../../config.js';
+
+// const createTestomatFormatter = apiKey => {
+//   if (!apiKey || apiKey === '') {
+//     console.log(pc.red('TESTOMATIO key is empty, ignoring reports'));
+//   }
+
+//   const documents = {};
+//   const dataTableMap = {};
+
+//   const addDocument = gherkinDocument => {
+//     documents[gherkinDocument.uri] = gherkinDocument.document;
+//   };
+
+//   const getTitle = scenario => {
+//     let { name } = scenario;
+//     if (scenario.tags.length) {
+//       let tags = '';
+//       for (const tag of scenario.tags) {
+//         tags = `${tags} ${tag.name}`;
+//       }
+//       name = `${name}${tags}`;
+//     }
+//     return name;
+//   };
+
+//   const getFeature = uri => documents[uri].feature;
+
+//   const getScenario = location => {
+//     const { children } = getFeature(location.uri);
+//     for (const scenario of children) {
+//       if (scenario.type === 'Scenario' && scenario.location.line === location.line) {
+//         return scenario;
+//       }
+//       if (scenario.type === 'ScenarioOutline') {
+//         for (const example of scenario.examples) {
+//           for (const tableBody of example.tableBody) {
+//             if (tableBody.location.line === location.line) {
+//               return scenario;
+//             }
+//           }
+//         }
+//       }
+//     }
+
+//     return null;
+//   };
+
+//   const loadDataTable = (scenario, uri) => {
+//     if (scenario.type === 'ScenarioOutline') {
+//       for (const example of scenario.examples) {
+//         for (const tableBody of example.tableBody) {
+//           const dataMap = example.tableHeader.cells.reduce((acc, cell, index) => {
+//             acc[cell.value] = tableBody.cells[index].value;
+//             return acc;
+//           }, {});
+
+//           dataTableMap[`${uri}:${tableBody.location.line}`] = JSON.stringify(dataMap);
+//         }
+//       }
+//     }
+//   };
+
+//   const getDataTableMap = (scenario, sourceLocation) => {
+//     const key = `${sourceLocation.uri}:${sourceLocation.line}`;
+//     if (!dataTableMap[key]) {
+//       loadDataTable(scenario, sourceLocation.uri);
+//     }
+
+//     return dataTableMap[key] || '';
+//   };
+
+//   const getTestId = scenario => {
+//     if (scenario) {
+//       for (const tag of scenario.tags) {
+//         const testId = getTestomatIdFromTestTitle(tag.name);
+//         if (testId) return testId;
+//       }
+//     }
+
+//     return null;
+//   };
+
+//   return class TestomatFormatter extends Formatter {
+//     constructor(options) {
+//       super(options);
+
+//       if (!apiKey) return;
+
+//       this.client = new TestomatClient({ apiKey });
+//       this.status = STATUS.PASSED.toString();
+
+//       options.eventBroadcaster.on('gherkin-document', addDocument);
+//       options.eventBroadcaster.on('test-run-started', () => {
+//         fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
+//         this?.client?.createRun();
+//       });
+//       options.eventBroadcaster.on('test-case-finished', this.onTestCaseFinished.bind(this));
+//       options.eventBroadcaster.on('test-case-started', this.onTestCaseStarted.bind(this));
+//       // @ts-ignore
+//       options.eventBroadcaster.on('test-run-finished', () => this?.client?.updateRunStatus(this.status));
+
+//       global.testomatioRunningEnvironment = 'cucumber:legacy';
+//     }
+
+//     onTestCaseStarted(event) {
+//       const scenario = getScenario(event.sourceLocation);
+//       const testId = getTestId(scenario);
+
+//       if (!global.testomatioDataStore) global.testomatioDataStore = {};
+//       global.testomatioDataStore.currentlyRunningTestId = testId;
+//     }
+
+//     onTestCaseFinished(event) {
+//       const scenario = getScenario(event.sourceLocation);
+//       const testId = getTestId(scenario);
+//       const status = event.result.status === 'undefined' ? STATUS.SKIPPED : event.result.status;
+
+//       let example = getDataTableMap(scenario, event.sourceLocation);
+//       if (example) example = JSON.parse(example);
+
+//       if (!scenario.name) return;
+
+//       const message = '';
+//       const cliMessage = `- ${scenario.name}: ${pc.bold(status.toUpperCase())}`;
+
+//       // if (event.result.status === 'undefined') {
+//       //   cliMessage += pc.yellow(
+//       //     ' (undefined steps. Run Cucumber without this formatter and implement missing steps)',
+//       //   );
+//       //   message = 'Undefined steps. Implement missing steps and rerun this scenario';
+//       // }
+//       console.log(cliMessage);
+//       if (status !== STATUS.PASSED && status !== STATUS.SKIPPED) {
+//         this.status = STATUS.FAILED;
+//       }
+
+//       this.client?.addTestRun(status, {
+//         error: event.result.exception instanceof Error ? event.result.exception : undefined,
+//         message,
+//         example,
+//         test_id: testId,
+//         title: getTitle(scenario),
+//         suite_title: getTitle(getFeature(event.sourceLocation.uri)),
+//       });
+//     }
+//   };
+// };
+
+// const CucumberLegacyReporter = createTestomatFormatter(config.TESTOMATIO);
+// export { CucumberLegacyReporter };
+// export default CucumberLegacyReporter;

package/src/adapter/cucumber.js

@@ -0,0 +1,4 @@
+import { CucumberReporter } from './cucumber/current.js';
+// import { CucumberLegacyReporter } from './cucumber/legacy.js';
+
+export default CucumberReporter;

package/src/adapter/cypress-plugin/index.js

@@ -0,0 +1,110 @@
+import { STATUS } from '../../constants.js';
+import { getTestomatIdFromTestTitle, parseSuite } from '../../utils/utils.js';
+import TestomatClient from '../../client.js';
+import { config } from '../../config.js';
+
+const testomatioReporter = on => {
+  if (!config.TESTOMATIO) {
+    console.log('TESTOMATIO key is empty, ignoring reports');
+    return;
+  }
+  const client = new TestomatClient({ apiKey: config.TESTOMATIO });
+
+  on('before:run', async () => {
+    // TODO: looks like client.env does not exist
+    // if (!client.env) {
+    //   client.env = `${run.browser.displayName},${run.system.osName}`;
+    // }
+    await client.createRun();
+  });
+
+  on('after:spec', async (_spec, results) => {
+    const addSpecTestsPromises = [];
+
+    const videos = [results.video];
+
+    for (const test of results.tests) {
+      const lastAttemptIndex = test.attempts.length - 1;
+      const latestAttempt = test.attempts[lastAttemptIndex];
+
+      // latestAttempt.duration && latestAttempt.error were available in adapters version up to 13 JFYI
+      const time = latestAttempt.duration || latestAttempt.wallClockDuration || test.duration;
+      let error = latestAttempt.error;
+
+      let title = test.title.pop();
+      const examples = title.match(/\(example (#\d+)\)/);
+      let example = null;
+      if (examples && examples[1]) example = { example: examples[1] };
+      title = title.replace(/\(example #\d+\)/, '').trim();
+
+      const suiteTitle = test.title.pop();
+
+      const testId = getTestomatIdFromTestTitle(title);
+      const suiteId = parseSuite(suiteTitle);
+
+      if (!error && test.displayError) {
+        error = { message: test.displayError };
+        error.inspect = function () {
+          return this.message;
+        };
+      }
+
+      const formattedError = error
+        ? {
+            message: error.message,
+            name: error.name,
+            inspect:
+              error.inspect ||
+              function () {
+                return this.message;
+              },
+          }
+        : undefined;
+
+      const screenshots = Array.isArray(results.screenshots)
+        ? results.screenshots
+            .filter(screenshot => screenshot?.path && screenshot?.path.includes(title) && screenshot?.takenAt)
+            .map(screenshot => screenshot.path)
+        : [];
+
+      const files = [...videos, ...screenshots];
+
+      let state;
+      switch (test.state) {
+        case 'passed':
+          state = STATUS.PASSED;
+          break;
+        case 'failed':
+          state = STATUS.FAILED;
+          break;
+        case 'skipped':
+        case 'pending':
+        default:
+          state = STATUS.SKIPPED;
+      }
+
+      addSpecTestsPromises.push(
+        client.addTestRun(state, {
+          title,
+          time,
+          example,
+          error: formattedError,
+          files,
+          suite_title: suiteTitle,
+          test_id: testId,
+          suite_id: suiteId,
+        }),
+      );
+    }
+
+    await Promise.all(addSpecTestsPromises);
+  });
+
+  on('after:run', async results => {
+    const status = results.totalFailed ? STATUS.FAILED : STATUS.PASSED;
+    // @ts-ignore
+    await client.updateRunStatus(status);
+  });
+};
+
+export default testomatioReporter;

package/src/adapter/jasmine.js

@@ -0,0 +1,60 @@
+import TestomatClient from '../client.js';
+import { getTestomatIdFromTestTitle, ansiRegExp } from '../utils/utils.js';
+import { STATUS } from '../constants.js';
+
+class JasmineReporter {
+  constructor(options) {
+    this.testTimeMap = {};
+    this.client = new TestomatClient({ apiKey: options?.apiKey });
+    this.client.createRun();
+  }
+
+  getDuration(test) {
+    if (this.testTimeMap[test.id]) {
+      return Date.now() - this.testTimeMap[test.id];
+    }
+
+    return 0;
+  }
+
+  specStarted(result) {
+    this.testTimeMap[result.id] = Date.now();
+  }
+
+  specDone(result) {
+    if (!this.client) return;
+
+    const title = result.description;
+    const { status } = result;
+    let errorMessage = '';
+
+    for (let i = 0; i < result.failedExpectations.length; i += 1) {
+      errorMessage = `${errorMessage}Failure: ${result.failedExpectations[i].message}\n`;
+      errorMessage = `${errorMessage}\n ${result.failedExpectations[i].stack}`;
+    }
+    console.log(`${title} : ${STATUS.PASSED}`);
+    console.log(errorMessage);
+    const testId = getTestomatIdFromTestTitle(title);
+    errorMessage = errorMessage.replace(ansiRegExp(), '');
+    this.client.addTestRun(status, {
+      error: result.failedExpectations[0],
+      message: errorMessage,
+      test_id: testId,
+      title,
+      time: this.getDuration(result),
+    });
+  }
+
+  jasmineDone(suiteInfo, done) {
+    if (!this.client) return;
+
+    const { overallStatus } = suiteInfo;
+    const status = overallStatus === 'failed' ? STATUS.FAILED : STATUS.PASSED;
+
+    // @ts-ignore
+    this.client.updateRunStatus(status).then(() => done);
+  }
+}
+
+export { JasmineReporter };
+export default JasmineReporter;

package/src/adapter/jest.js

@@ -0,0 +1,108 @@
+import pc from 'picocolors';
+import TestomatClient from '../client.js';
+import { TESTOMAT_TMP_STORAGE_DIR } from '../constants.js';
+import { getTestomatIdFromTestTitle, ansiRegExp, fileSystem } from '../utils/utils.js';
+import { services } from '../services/index.js';
+import path from 'path';
+import createDebugMessages from 'debug';
+
+const debug = createDebugMessages('@testomatio/reporter:adapter-jest');
+
+export class JestReporter {
+  constructor(globalConfig, options) {
+    this._globalConfig = globalConfig;
+    this._options = options;
+
+    this.client = new TestomatClient({ apiKey: options?.apiKey });
+    this.client.createRun();
+  }
+
+  onRunStart() {
+    // clear tmp dir
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
+  }
+
+  // start of test file (including beforeAll)
+  onTestStart(testFile) {
+    debug('Start running test file:', testFile.path);
+    services.setContext(testFile.path);
+  }
+
+  // start of the test (including beforeEach)
+  onTestCaseStart(test, testCase) {
+    debug('Start running test:', testCase.fullName);
+    services.setContext(testCase.fullName);
+  }
+
+  // end of test file! (there is also onTestCaseResult listener)
+  onTestResult(test, testResult) {
+    if (!this.client) return;
+
+    const { testResults } = testResult;
+    for (const result of testResults) {
+      let error;
+      let steps;
+      const { status, title, duration, failureMessages } = result;
+      if (failureMessages[0]) {
+        const errorMessage = failureMessages[0].replace(ansiRegExp(), '');
+        error = new Error(errorMessage);
+        steps = failureMessages[0];
+      }
+      const testId = getTestomatIdFromTestTitle(title);
+
+      // suite titles from most outer to most inner, separated by space
+      let fullSuiteTitle = testResult.ancestorTitles?.join(' ');
+      // if no suite titles, use file name
+      if (!fullSuiteTitle && testResult.testFilePath) fullSuiteTitle = path.basename(testResult.testFilePath);
+
+      const logs = getTestLogs(result);
+      const artifacts = services.artifacts.get(result.fullName);
+      const keyValues = services.keyValues.get(result.fullName);
+      const links = services.links.get(result.fullName);
+
+      const deducedStatus = status === 'pending' ? 'skipped' : status;
+      // In jest if test is not matched with test name pattern it is considered as skipped.
+      // So adding a check if it is skipped for real or because of test pattern
+      if (!this._globalConfig.testNamePattern || deducedStatus !== 'skipped') {
+        this.client.addTestRun(deducedStatus, {
+          test_id: testId,
+          suite_title: fullSuiteTitle,
+          error,
+          steps,
+          title,
+          time: duration,
+          logs,
+          links,
+          manuallyAttachedArtifacts: artifacts,
+          meta: keyValues,
+        });
+      }
+    }
+  }
+
+  onRunComplete(contexts, results) {
+    if (!this.client) return;
+
+    const { numFailedTests } = results;
+    const status = numFailedTests === 0 ? 'passed' : 'failed';
+    this.client.updateRunStatus(status);
+  }
+}
+
+function getTestLogs(testResult) {
+  const suiteLogsArr = services.logger.getLogs(testResult.testFilePath);
+  const suiteLogs = suiteLogsArr ? suiteLogsArr.join('\n').trim() : '';
+  const testLogsArr = services.logger.getLogs(testResult.fullName);
+  const testLogs = testLogsArr ? testLogsArr.join('\n').trim() : '';
+
+  let logs = '';
+  if (suiteLogs) {
+    logs += `${pc.bold('\t--- Suite ---')}\n${suiteLogs}`;
+  }
+  if (testLogs) {
+    logs += `\n${pc.bold('\t--- Test ---')}\n${testLogs}`;
+  }
+  return logs;
+}
+
+export default JestReporter;

package/src/adapter/mocha.cjs

@@ -0,0 +1,2 @@
+const MochaReporter = require('../../../lib/adapter/mocha.js');
+module.exports = MochaReporter;