testomatio-reporter-cli 2.8.4 → 2.8.5-beta.2-yarn

package/src/adapter/mocha.js

@@ -0,0 +1,211 @@
+import Mocha from 'mocha';
+import TestomatClient from '../client.js';
+import { STATUS, TESTOMAT_TMP_STORAGE_DIR } from '../constants.js';
+import { getTestomatIdFromTestTitle, fileSystem } from '../utils/utils.js';
+import { config } from '../config.js';
+import { services } from '../services/index.js';
+import pc from 'picocolors';
+
+const {
+  EVENT_RUN_BEGIN,
+  EVENT_RUN_END,
+  EVENT_TEST_FAIL,
+  EVENT_TEST_PASS,
+  EVENT_TEST_PENDING,
+  EVENT_SUITE_BEGIN,
+  EVENT_SUITE_END,
+  EVENT_TEST_BEGIN,
+  EVENT_TEST_END,
+} = Mocha.Runner.constants;
+
+function MochaReporter(runner, opts) {
+  Mocha.reporters.Base.call(this, runner);
+  let passes = 0;
+  let failures = 0;
+  let skipped = 0;
+  // let artifactStore;
+
+  const apiKey = opts?.reporterOptions?.apiKey || config.TESTOMATIO;
+
+  const client = new TestomatClient({ apiKey });
+
+  // Track hook failures
+  const hookFailures = new Map();
+
+  runner.on(EVENT_RUN_BEGIN, () => {
+    client.createRun();
+
+    // clear dir with artifacts/logs
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
+  });
+
+  runner.on(EVENT_SUITE_BEGIN, async suite => {
+    services.setContext(suite.fullTitle());
+  });
+
+  runner.on(EVENT_SUITE_END, async suite => {
+    if (hookFailures.has(suite.fullTitle())) {
+      const { error, suiteTitle } = hookFailures.get(suite.fullTitle());
+
+      for (const test of suite.tests) {
+        if (test.state === 'pending' || !test.state) {
+          const testId = getTestomatIdFromTestTitle(test.title);
+          const artifacts = services.artifacts.get(test.fullTitle());
+          const keyValues = services.keyValues.get(test.fullTitle());
+          const links = services.links.get(test.fullTitle());
+
+          client.addTestRun(STATUS.FAILED, {
+            error,
+            suite_title: suiteTitle || suite.title,
+            file: suite.file,
+            test_id: testId,
+            title: test.title,
+            code: process.env.TESTOMATIO_UPDATE_CODE ? test.body.toString() : '',
+            time: 0,
+            manuallyAttachedArtifacts: artifacts,
+            meta: keyValues,
+            links,
+          });
+        }
+      }
+    }
+
+    services.setContext(null);
+  });
+
+  runner.on(EVENT_TEST_BEGIN, async test => {
+    services.setContext(test.fullTitle());
+  });
+
+  runner.on(EVENT_TEST_END, async () => {
+    services.setContext(null);
+  });
+
+  runner.on(EVENT_TEST_PASS, async test => {
+    passes += 1;
+
+    console.log(pc.bold(pc.green('✔')), test.fullTitle());
+    const testId = getTestomatIdFromTestTitle(test.title);
+
+    const logs = getTestLogs(test);
+    const artifacts = services.artifacts.get(test.fullTitle());
+    const keyValues = services.keyValues.get(test.fullTitle());
+    const links = services.links.get(test.fullTitle());
+
+    client.addTestRun(STATUS.PASSED, {
+      test_id: testId,
+      suite_title: getSuiteTitle(test),
+      title: getTestName(test),
+      code: process.env.TESTOMATIO_UPDATE_CODE ? test.body.toString() : '',
+      file: getFile(test),
+      time: test.duration,
+      logs,
+      manuallyAttachedArtifacts: artifacts,
+      meta: keyValues,
+      links,
+    });
+  });
+
+  runner.on(EVENT_TEST_PENDING, test => {
+    skipped += 1;
+    console.log('skip: %s', test.fullTitle());
+    const testId = getTestomatIdFromTestTitle(test.title);
+    const artifacts = services.artifacts.get(test.fullTitle());
+    const keyValues = services.keyValues.get(test.fullTitle());
+    const links = services.links.get(test.fullTitle());
+
+    client.addTestRun(STATUS.SKIPPED, {
+      title: getTestName(test),
+      suite_title: getSuiteTitle(test),
+      code: process.env.TESTOMATIO_UPDATE_CODE ? test.body.toString() : '',
+      file: getFile(test),
+      test_id: testId,
+      time: test.duration,
+      manuallyAttachedArtifacts: artifacts,
+      meta: keyValues,
+      links,
+    });
+  });
+
+  runner.on(EVENT_TEST_FAIL, async (test, err) => {
+    failures += 1;
+    console.log(pc.bold(pc.red('✖')), test.fullTitle(), pc.gray(err.message));
+
+    const isHookFailure = test.title.includes('before each') || test.title.includes('after each');
+
+    if (isHookFailure && test.parent) {
+      hookFailures.set(test.parent.fullTitle(), {
+        error: err,
+        suiteTitle: getSuiteTitle(test),
+      });
+      return;
+    }
+
+    const testId = getTestomatIdFromTestTitle(test.title);
+
+    const logs = getTestLogs(test);
+    const artifacts = services.artifacts.get(test.fullTitle());
+    const keyValues = services.keyValues.get(test.fullTitle());
+    const links = services.links.get(test.fullTitle());
+
+    client.addTestRun(STATUS.FAILED, {
+      error: err,
+      suite_title: getSuiteTitle(test),
+      file: getFile(test),
+      test_id: testId,
+      title: getTestName(test),
+      code: process.env.TESTOMATIO_UPDATE_CODE ? test.body.toString() : '',
+      time: test.duration,
+      logs,
+      manuallyAttachedArtifacts: artifacts,
+      meta: keyValues,
+      links,
+    });
+  });
+
+  runner.on(EVENT_RUN_END, () => {
+    const status = failures === 0 ? STATUS.PASSED : STATUS.FAILED;
+    console.log(pc.bold(status), `${passes} passed, ${failures} failed, ${skipped} skipped`);
+    // @ts-ignore
+    client.updateRunStatus(status);
+  });
+}
+
+function getTestLogs(test) {
+  const suiteLogsArr = services.logger.getLogs(test.parent.fullTitle());
+  const suiteLogs = suiteLogsArr ? suiteLogsArr.join('\n').trim() : '';
+  const testLogsArr = services.logger.getLogs(test.fullTitle());
+  const testLogs = testLogsArr ? testLogsArr.join('\n').trim() : '';
+
+  let logs = '';
+  if (suiteLogs) {
+    logs += `${pc.bold('\t--- BeforeSuite ---')}\n${suiteLogs}`;
+  }
+  if (testLogs) {
+    logs += `\n${pc.bold('\t--- Test ---')}\n${testLogs}`;
+  }
+  return logs;
+}
+
+function getSuiteTitle(test, pathArr = []) {
+  if (test.parent.parent) getSuiteTitle(test.parent, pathArr);
+
+  pathArr.push(test.parent.title);
+
+  return pathArr.filter(t => !!t)[0];
+}
+
+function getFile(test) {
+  return test.parent.file?.replace(process.cwd(), '');
+}
+
+function getTestName(test) {
+  if (process.env.TESTOMATIO_CREATE === 'fulltitle') return test.fullTitle();
+  return test.title;
+}
+
+// To have this reporter "extend" a built-in reporter uncomment the following line:
+// @ts-ignore
+Mocha.utils.inherits(MochaReporter, Mocha.reporters.Spec);
+
+export default MochaReporter;

package/src/adapter/nightwatch.js

@@ -0,0 +1,88 @@
+import TestomatClient from '../client.js';
+import { config } from '../config.js';
+import { STATUS } from '../constants.js';
+import { getTestomatIdFromTestTitle } from '../utils/utils.js';
+
+const apiKey = config.TESTOMATIO;
+const client = new TestomatClient({ apiKey });
+
+export default {
+  write: async (results, options, done) => {
+    await client.createRun();
+
+    const testFiles = results.modules;
+
+    for (const fileName in testFiles) {
+      // in nightwatch: object containing tests from a single file
+      const testModule = testFiles[fileName];
+
+      // passed and failed tests (tests with assertions)
+      const completedTests = testModule.completed;
+
+      // skipped tests (skipped by user or tests without assertions)
+      const skippedTests = testModule.skipped;
+
+      const tags = testModule.tags || [];
+
+      // if test file contains multiple suites, the last suite name is used as a name 🤷‍♂️
+      // no other places which contain suite name (even inside test object)
+      const suiteTitle = testModule.name;
+
+      for (const testTitle in completedTests) {
+        const test = completedTests[testTitle];
+        let status;
+        switch (test.status) {
+          case 'pass':
+            status = STATUS.PASSED;
+            break;
+          case 'fail':
+            status = STATUS.FAILED;
+            break;
+          // probably not required (because skipped tests are in separate array), but just in case
+          case 'skip':
+            status = STATUS.SKIPPED;
+            console.info('Skipped test is in completed tests array:', test, 'Not expected behavior.');
+            break;
+          default:
+            console.error('Test status processing error:', test.status);
+        }
+
+        const testId = getTestomatIdFromTestTitle(testTitle);
+
+        client.addTestRun(status, {
+          error: { name: test.assertions?.[0]?.name, message: test.assertions?.[0]?.message, stack: test.stackTrace },
+          file: testModule.modulePath?.replace(process.cwd(), ''),
+          message: test.assertions?.[0]?.message,
+          rid: `${testModule.uuid || ''}_${testTitle || ''}`,
+          stack: test.stackTrace,
+          suite_title: suiteTitle,
+          tags,
+          test_id: testId,
+          time: test.timeMs,
+          title: testTitle,
+        });
+      }
+
+      // just array with skipped tests titles, no any other info
+      for (const testTitle of skippedTests) {
+        client.addTestRun(STATUS.SKIPPED, {
+          suite_title: suiteTitle,
+          tags,
+          rid: `${testModule.uuid || ''}_${testTitle || ''}`,
+          title: testTitle,
+        });
+      }
+    }
+
+    /**
+     * @type {'passed' | 'failed' | 'finished'}
+     */
+    let runStatus = 'finished';
+    if (results.failed) runStatus = 'failed';
+    else if (results.passed) runStatus = 'passed';
+
+    await client.updateRunStatus(runStatus);
+
+    done();
+  },
+};

package/src/adapter/playwright.js

@@ -0,0 +1,343 @@
+import crypto from 'crypto';
+import os from 'os';
+import path from 'path';
+import { v4 as uuidv4 } from 'uuid';
+import fs from 'fs';
+import { APP_PREFIX, STATUS as Status, TESTOMAT_TMP_STORAGE_DIR, SCREENSHOTS_ON_STEPS } from '../constants.js';
+import TestomatioClient from '../client.js';
+import { getTestomatIdFromTestTitle, fileSystem, truncate } from '../utils/utils.js';
+import { services } from '../services/index.js';
+import { dataStorage } from '../data-storage.js';
+import { extensionMap } from '../utils/constants.js';
+import pc from 'picocolors';
+import { fetchLinksFromLogs } from './utils/playwright.js';
+import { formatStep, addStatusToStep, addArtifactsToStep } from './utils/step-formatter.js';
+import { log } from '../utils/log.js';
+
+const reportTestPromises = [];
+
+class PlaywrightReporter {
+  constructor(config = {}) {
+    this.client = new TestomatioClient({ apiKey: config?.apiKey });
+
+    this.uploads = [];
+  }
+
+  onBegin(config, suite) {
+    // clean data storage
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
+    if (!this.client) return;
+    this.suite = suite;
+    this.config = config;
+    this.client.createRun();
+  }
+
+  onTestBegin(testInfo) {
+    const fullTestTitle = getTestContextName(testInfo);
+    dataStorage.setContext(fullTestTitle);
+  }
+
+  async onTestEnd(test, result) {
+    // test.parent.project().__projectId
+
+    if (!this.client) return;
+
+    const { title } = test;
+    const { error, duration } = result;
+    const pwAttachments = (result.attachments || []).filter(a => a.body || a.path);
+
+    const files = pwAttachments
+      .map(att => ({
+        path: this.#getArtifactPath(att),
+        title: att.name || title,
+        type: att.contentType,
+      }))
+      .filter(f => f.path);
+
+    const suite_title = test.parent ? test.parent?.title : path.basename(test?.location?.file);
+
+    const rid = test.id || test.testId || uuidv4();
+
+    /**
+     * @type {{
+     * browser?: string,
+     * dependencies: string[],
+     * isMobile?: boolean
+     * metadata: Record<string, any>,
+     * name: string,
+     * }}
+     */
+    const project = {
+      browser: test.parent.project().use.defaultBrowserType,
+      dependencies: test.parent.project().dependencies,
+      isMobile: test.parent.project().use.isMobile,
+      metadata: test.parent.project().metadata,
+      name: test.parent.project().name,
+    };
+
+    const steps = result.steps.map(step => appendStep(step, 0)).filter(step => step !== null);
+
+    // Extract and normalize tags
+    const tags = extractTags(test);
+
+    const fullTestTitle = getTestContextName(test);
+
+    let logs = '';
+    // get links along with filtered logs (liks related logs removed)
+    const { stdout: filteredStdout, links, meta } = fetchLinksFromLogs(result.stdout);
+    if (filteredStdout?.length || result.stderr?.length) {
+      logs = `\n\n${pc.bold('Logs:')}\n${pc.red(result.stderr.join(''))}\n${filteredStdout.join('')}`;
+    }
+
+    /*
+      All services fucntions work different for Playwright.
+      We don't have access to test title (as result, to test id) when calling this functions inside a test.
+      Thus, when user calls services functions inside a test, we just log this data to console.
+      Playwright intercepts the console.log on it's end and we just get this data from it.
+      Thus, we have a tiny drawback: all data from services functions inside a test will be logged to console.
+      And this requires a condition to be added for each service function – if its Playwright, then log to console.
+
+      "get" method of services will not return data for Playwright, we should parse stdout.
+    */
+    const manuallyAttachedArtifacts = services.artifacts.get(fullTestTitle);
+    const testMeta = services.keyValues.get(fullTestTitle);
+
+    let status = result.status;
+    // process test.fail() annotation
+    if (test.expectedStatus === 'failed') {
+      // actual status = expected
+      if (result.status === 'failed') status = 'passed';
+      // actual status != expected
+      if (result.status === 'passed') status = 'failed';
+    }
+
+    const reportTestPromise = this.client.addTestRun(checkStatus(status), {
+      rid: `${rid}-${project.name}`,
+      error,
+      test_id: getTestomatIdFromTestTitle(`${title} ${tags.join(' ')}`),
+      suite_title,
+      title,
+      tags: tags.map(tag => tag.replace('@', '')),
+      steps: steps.length ? steps : undefined,
+      time: duration,
+      logs,
+      links,
+      manuallyAttachedArtifacts,
+      files: files.length ? files : undefined,
+      meta: {
+        browser: project.browser,
+        isMobile: project.isMobile,
+        project: project.name,
+        projectDependencies: project.dependencies?.length ? project.dependencies : null,
+        ...testMeta,
+        ...meta,
+        ...project.metadata, // metadata has any type (in playwright), but we will stringify it in client.js
+        ...test.annotations?.reduce((acc, annotation) => {
+          acc[annotation.type] = annotation.description;
+          return acc;
+        }, {}),
+      },
+      file: test.location?.file,
+    });
+
+    this.uploads.push({
+      rid: `${rid}-${project.name}`,
+      title: test.title,
+      files: pwAttachments,
+      file: test.location?.file,
+    });
+    // remove empty uploads
+    this.uploads = this.uploads.filter(anUpload => anUpload.files.length);
+
+    reportTestPromises.push(reportTestPromise);
+  }
+
+  #getArtifactPath(artifact) {
+    if (artifact.path) {
+      if (path.isAbsolute(artifact.path)) return artifact.path;
+      return path.join(this.config.outputDir || this.config.projects[0].outputDir, artifact.path);
+    }
+    if (artifact.body) {
+      let filePath = generateTmpFilepath(artifact.name);
+      const hasExtension = artifact.name && path.extname(artifact.name);
+      if (!hasExtension && artifact.contentType) {
+        const extension = extensionMap[artifact.contentType] || artifact.contentType.split('/')[1];
+        if (extension) filePath += `.${extension}`;
+      }
+      fs.writeFileSync(filePath, artifact.body);
+      return filePath;
+    }
+    return null;
+  }
+
+  async onEnd(result) {
+    if (!this.client) return;
+
+    await Promise.all(reportTestPromises);
+
+    if (this.uploads.length) {
+      if (this.client.uploader.isEnabled) log.info(`🎞️ Uploading ${this.uploads.length} files...`);
+
+      const promises = [];
+
+      // ? possible move to addTestRun (needs investigation if files are ready)
+      for (const upload of this.uploads) {
+        const { rid, file, title } = upload;
+
+        const files = upload.files.map(attachment => ({
+          path: this.#getArtifactPath(attachment),
+          title,
+          type: attachment.contentType,
+        }));
+
+        if (!this.client.uploader.isEnabled) {
+          files.forEach(f => this.client.uploader.storeUploadedFile(f, this.client.runId, rid, false));
+          continue;
+        }
+
+        promises.push(
+          this.client.addTestRun(undefined, {
+            rid,
+            title,
+            files,
+            file,
+          }),
+        );
+      }
+      await Promise.all(promises);
+    }
+
+    await this.client.updateRunStatus(checkStatus(result.status));
+  }
+}
+
+function checkStatus(status) {
+  return (
+    {
+      skipped: Status.SKIPPED,
+      timedOut: Status.FAILED,
+      passed: Status.PASSED,
+    }[status] || Status.FAILED
+  );
+}
+
+function appendStep(step, shift = 0) {
+  // nesting too deep, ignore those steps
+  if (shift >= 10) return;
+
+  let newCategory = step.category;
+  switch (newCategory) {
+    case 'test.step':
+      newCategory = 'user';
+      break;
+    case 'hook':
+      newCategory = 'hook';
+      break;
+    case 'attach':
+      return null; // Skip steps with category 'attach'
+    default:
+      newCategory = 'framework';
+  }
+
+  const resultStep = formatStep({
+    category: newCategory,
+    title: step.title,
+    duration: step.duration,
+  });
+
+  // Add status based on error
+  addStatusToStep(resultStep, step.error ? 'failed' : 'passed', step.error);
+
+  // Add error if present
+  if (step.error !== undefined) {
+    if (typeof step.error === 'object') {
+      resultStep.error = {
+        message: truncate(String(step.error.message), 250),
+        stack: truncate(String(step.error.stack || ''), 250),
+      };
+    } else {
+      resultStep.error = truncate(String(step.error), 250);
+    }
+  }
+
+  // Add log if present
+  if (step.log) {
+    resultStep.log = truncate(String(step.log), 250);
+  }
+
+  // Add artifacts from attachments
+  if (step.attachments && step.attachments.length > 0 && SCREENSHOTS_ON_STEPS) {
+    const screenshotAttachment = step.attachments.find(att =>
+      att.contentType === 'image/png' && att.name === 'screenshot'
+    );
+    if (screenshotAttachment && screenshotAttachment.path) {
+      const artifacts = { screenshot: screenshotAttachment.path };
+      addArtifactsToStep(resultStep, artifacts);
+    }
+  }
+
+  // Process nested steps
+  const formattedSteps = [];
+  for (const child of step.steps || []) {
+    const appendedChild = appendStep(child, shift + 2);
+    if (appendedChild) {
+      formattedSteps.push(appendedChild);
+    }
+  }
+
+  if (formattedSteps.length) {
+    resultStep.steps = formattedSteps.filter(s => !!s);
+  }
+
+  return resultStep;
+}
+
+function generateTmpFilepath(filename = '') {
+  filename = filename || `tmp.${crypto.randomBytes(16).toString('hex')}`;
+  const tmpdir = os.tmpdir();
+  return path.join(tmpdir, filename);
+}
+
+/**
+ * Extracts tags from test title, test options, and suite level
+ * Identifies duplicate tags (case-insensitive)
+ * @param {*} test - testInfo object from Playwright
+ * @returns {string[]} - array of normalized tags with @ prefix
+ */
+function extractTags(test) {
+  const tagsMap = new Map(); // key: lowercase tag, value: original case tag
+
+  function addTag(tag) {
+    if (typeof tag !== 'string') return;
+    const trimmed = tag.trim();
+    if (!trimmed) return;
+    const normalizedTag = trimmed.startsWith('@') ? trimmed : `@${trimmed}`;
+    const lowercaseTag = normalizedTag.toLowerCase();
+    if (!tagsMap.has(lowercaseTag)) {
+      tagsMap.set(lowercaseTag, normalizedTag);
+    }
+  }
+
+  // Extract tags from test title (@tag format); only test title is considered
+  const titleTagsMatch = test.title.match(/@[A-Za-z0-9_-]+/g) || [];
+  titleTagsMatch.forEach(addTag);
+
+  // Extract tags from test.tags (Playwright built-in tags); ignore parents
+  if (Array.isArray(test.tags)) {
+    test.tags.forEach(addTag);
+  }
+
+  return Array.from(tagsMap.values());
+}
+
+/**
+ * Returns filename + test title
+ * @param {*} test - testInfo object from Playwright
+ * @returns
+ */
+function getTestContextName(test) {
+  return `${test._requireFile || ''}_${test.title}`;
+}
+
+export default PlaywrightReporter;
+export { extractTags, fetchLinksFromLogs };