testomatio-reporter-cli 2.8.5-beta.2-yarn → 2.9.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testomatio-reporter-cli",
-  "version": "2.8.5-beta.2-yarn",
+  "version": "2.9.0",
   "description": "Yarn Berry compatible standalone CLI for @testomatio/reporter",
   "type": "module",
   "bin": {

package/src/adapter/codecept.js

@@ -48,6 +48,7 @@ if (MAJOR_VERSION === 3 && MINOR_VERSION < 7) {
 
 function CodeceptReporter(config) {
   const failedTests = [];
+  const reportedTestUids = new Set();
   let videos = [];
   let traces = [];
   const reportTestPromises = [];
@@ -161,6 +162,7 @@ function CodeceptReporter(config) {
     const error = hook?.ctx?.currentTest?.err;
 
     for (const test of suite.tests) {
+      reportedTestUids.add(test.uid);
       const reportTestPromise = client.addTestRun('failed', {
         ...stripExampleFromTitle(test.title),
         rid: test.uid,
@@ -197,9 +199,29 @@ function CodeceptReporter(config) {
     });
   });
 
+  event.dispatcher.on(event.test.skipped, test => {
+    const { uid, tags, title } = test.simplify();
+
+    if (uid && reportedTestUids.has(uid)) return;
+
+    services.setContext(null);
+
+    const reportTestPromise = client.addTestRun(STATUS.SKIPPED, {
+      ...stripExampleFromTitle(title),
+      rid: uid,
+      test_id: getTestomatIdFromTestTitle(`${title} ${tags?.join(' ')}`),
+      suite_title: test.parent && stripTagsFromTitle(stripExampleFromTitle(test.parent.title).title),
+      time: test.duration,
+      meta: test.meta,
+    });
+    reportTestPromises.push(reportTestPromise);
+    reportedTestUids.add(uid);
+  });
+
   event.dispatcher.on(event.test.after, test => {
     const { uid, tags, title, artifacts } = test.simplify();
     const error = test.err || null;
+    reportedTestUids.add(uid);
     failedTests.push(uid || title);
     const testObj = getTestAndMessage(title);
     const files = buildArtifactFiles(artifacts);
@@ -211,8 +233,8 @@ function CodeceptReporter(config) {
 
     // Build step hierarchy with screenshot from screenshotOnFail
     const stepHierarchy = buildUnifiedStepHierarchy(
-      test.steps, 
-      hookSteps, 
+      test.steps,
+      hookSteps,
       screenshotOnFailPath
     );
 

package/src/adapter/playwright.js

@@ -14,13 +14,13 @@ import { fetchLinksFromLogs } from './utils/playwright.js';
 import { formatStep, addStatusToStep, addArtifactsToStep } from './utils/step-formatter.js';
 import { log } from '../utils/log.js';
 
-const reportTestPromises = [];
-
 class PlaywrightReporter {
   constructor(config = {}) {
     this.client = new TestomatioClient({ apiKey: config?.apiKey });
 
     this.uploads = [];
+    this.reportTestPromises = [];
+    this.runPromise = Promise.resolve();
   }
 
   onBegin(config, suite) {
@@ -29,7 +29,9 @@ class PlaywrightReporter {
     if (!this.client) return;
     this.suite = suite;
     this.config = config;
-    this.client.createRun();
+    this.uploads = [];
+    this.reportTestPromises = [];
+    this.runPromise = this.client.createRun();
   }
 
   onTestBegin(testInfo) {
@@ -41,6 +43,7 @@ class PlaywrightReporter {
     // test.parent.project().__projectId
 
     if (!this.client) return;
+    await this.runPromise;
 
     const { title } = test;
     const { error, duration } = result;
@@ -133,7 +136,11 @@ class PlaywrightReporter {
         ...meta,
         ...project.metadata, // metadata has any type (in playwright), but we will stringify it in client.js
         ...test.annotations?.reduce((acc, annotation) => {
-          acc[annotation.type] = annotation.description;
+          if (acc[annotation.type]) {
+            acc[annotation.type] = `${acc[annotation.type]}, ${annotation.description}`;
+          } else {
+            acc[annotation.type] = annotation.description;
+          }
           return acc;
         }, {}),
       },
@@ -149,7 +156,7 @@ class PlaywrightReporter {
     // remove empty uploads
     this.uploads = this.uploads.filter(anUpload => anUpload.files.length);
 
-    reportTestPromises.push(reportTestPromise);
+    this.reportTestPromises.push(reportTestPromise);
   }
 
   #getArtifactPath(artifact) {
@@ -173,7 +180,8 @@ class PlaywrightReporter {
   async onEnd(result) {
     if (!this.client) return;
 
-    await Promise.all(reportTestPromises);
+    await this.runPromise;
+    await Promise.all(this.reportTestPromises);
 
     if (this.uploads.length) {
       if (this.client.uploader.isEnabled) log.info(`🎞️ Uploading ${this.uploads.length} files...`);

package/src/bin/cli.js

@@ -6,7 +6,7 @@ import { glob } from 'glob';
 import createDebugMessages from 'debug';
 import TestomatClient from '../client.js';
 import XmlReader from '../xmlReader.js';
-import { APP_PREFIX, STATUS, DEBUG_FILE } from '../constants.js';
+import { APP_PREFIX, STATUS, DEBUG_FILE, BATCH_MODE } from '../constants.js';
 import { cleanLatestRunId, getPackageVersion, applyFilter } from '../utils/utils.js';
 import { config } from '../config.js';
 import { readLatestRunId } from '../utils/utils.js';
@@ -369,7 +369,7 @@ program
     const client = new TestomatClient({
       apiKey,
       runId,
-      isBatchEnabled: false,
+      batchMode: BATCH_MODE.DISABLED,
     });
 
     let testruns = client.uploader.readUploadedFiles(runId);

package/src/bin/uploadArtifacts.js

@@ -9,6 +9,7 @@ import { config } from '../config.js';
 import { readLatestRunId } from '../utils/utils.js';
 import dotenv from 'dotenv';
 import { log } from '../utils/log.js';
+import { BATCH_MODE } from '../constants.js';
 
 const debug = createDebugMessages('@testomatio/reporter:upload-cli');
 const version = getPackageVersion();
@@ -37,7 +38,7 @@ program
     const client = new TestomatClient({
       apiKey,
       runId,
-      isBatchEnabled: false,
+      batchMode: BATCH_MODE.DISABLED,
     });
     let testruns = client.uploader.readUploadedFiles(process.env.TESTOMATIO_RUN);
 

package/src/constants.js

@@ -28,6 +28,14 @@ const STATUS = {
   SKIPPED: 'skipped',
   FINISHED: 'finished',
 };
+
+// batch upload mode
+/** @type {{ AUTO: 'auto', MANUAL: 'manual', DISABLED: 'disabled' }} */
+const BATCH_MODE = {
+  AUTO: 'auto',
+  MANUAL: 'manual',
+  DISABLED: 'disabled',
+};
 // html pipe var
 const HTML_REPORT = {
   FOLDER: 'html-report',
@@ -61,6 +69,7 @@ export {
   TESTOMAT_TMP_STORAGE_DIR,
   CSV_HEADERS,
   STATUS,
+  BATCH_MODE,
   HTML_REPORT,
   MARKDOWN_REPORT,
   REQUEST_TIMEOUT,

package/src/pipe/coverage.js

@@ -261,9 +261,10 @@ class CoveragePipe { // or Changes for the future???
      */
     #getChangedFilesFromGit(cmd) {
         try {
+            // Capture stderr (instead of ignoring it) so Git's actual error is available for diagnostics
             const result = execSync(cmd, {
                 encoding: 'utf-8',
-                stdio: ['pipe', 'pipe', 'ignore']
+                stdio: ['pipe', 'pipe', 'pipe']
             });
 
             return result
@@ -272,16 +273,33 @@ class CoveragePipe { // or Changes for the future???
                 .filter(Boolean);
         }
         catch (err) {
-            const errorMessage = err.message || '';
-            // Git edge: Not a git repository or other error
+            // Prefer Git's own stderr output, fall back to the generic error message
+            const gitOutput = (err.stderr || '').toString().trim();
+            const errorMessage = gitOutput || err.message || '';
+
+            // Git edge: Not a git repository
             if (errorMessage.includes('Not a git repository')) {
-                log.error( '❌ Error: This folder is not a Git repository.');
+                log.error('❌ Error: This folder is not a Git repository.');
+                return [];
             }
-            else {
-                throw new Error(`❌ Git command failed ("${cmd}"):\n`, errorMessage);
+
+            // Git edge: the branch/ref to diff against is not available locally.
+            // This is common in CI, where a shallow checkout fetches only the current branch.
+            if (
+                errorMessage.includes('unknown revision') ||
+                errorMessage.includes('ambiguous argument') ||
+                errorMessage.includes('bad revision')
+            ) {
+                log.error(`❌ Git command failed ("${cmd}"):\n${errorMessage}`);
+                log.error(
+                    `🔍 Branch "${this.branch}" was not found locally. ` +
+                    `In CI this usually means a shallow checkout — fetch full history first, e.g. ` +
+                    `actions/checkout with "fetch-depth: 0", or run "git fetch origin ${this.branch}:${this.branch}".`
+                );
+                return [];
             }
 
-            return [];
+            throw new Error(`❌ Git command failed ("${cmd}"):\n${errorMessage}`);
         }
     }
 

package/src/pipe/debug.js

@@ -14,13 +14,7 @@ export class DebugPipe {
 
     this.isEnabled = !!process.env.TESTOMATIO_DEBUG || !!process.env.DEBUG;
     if (this.isEnabled) {
-      this.batch = {
-        isEnabled: this.params.isBatchEnabled ?? !process.env.TESTOMATIO_DISABLE_BATCH_UPLOAD,
-        intervalFunction: null,
-        intervalTime: 5000,
-        tests: [],
-        batchIndex: 0,
-      };
+      this.tests = [];
       const suffix = process.env.TESTOMATIO_REPLAY ? 'replay' : '';
       const paths = getDebugFilePath(suffix);
       this.logFilePath = paths.tmp;
@@ -60,8 +54,13 @@ export class DebugPipe {
       this.logToFile({ datetime: new Date().toISOString(), timestamp: Date.now() });
       this.logToFile({ data: 'variables', testomatioEnvVars: this.testomatioEnvVars });
       this.logToFile({ data: 'store', store: this.store || {} });
-      // Bind batchUpload to the instance
-      this.batchUpload = this.batchUpload.bind(this);
+
+      // Safety net for hook failures (e.g. a failing AfterSuite) that abort the run
+      // before finishRun: buffered tests would otherwise be lost. The handler is
+      // attached lazily when the first test is buffered and detached once flushed,
+      // so processes that create many pipes don't pile up `exit` listeners.
+      this.flushOnExit = () => this.flushBufferedTests();
+      this.exitListenerAttached = false;
     }
   }
 
@@ -88,42 +87,22 @@ export class DebugPipe {
 
   async createRun(params = {}) {
     if (!this.isEnabled) return;
-    if (params.isBatchEnabled === true || params.isBatchEnabled === false) this.batch.isEnabled = params.isBatchEnabled;
-
-    if (!this.isEnabled) return {};
-    if (this.batch.isEnabled) this.batch.intervalFunction = setInterval(this.batchUpload, this.batch.intervalTime);
 
     this.logToFile({ action: 'createRun', params });
   }
 
   async addTest(data) {
     if (!this.isEnabled) return;
-
-    if (!this.batch.isEnabled) {
-      const logData = { action: 'addTest', testId: data };
-      if (this.store.runId) logData.runId = this.store.runId;
-      this.logToFile(logData);
-    } else this.batch.tests.push(data);
-
-    if (!this.batch.intervalFunction) await this.batchUpload();
-  }
-
-  async batchUpload() {
-    this.batch.batchIndex++;
-    if (!this.batch.isEnabled) return;
-    if (!this.batch.tests.length) return;
-
-    const testsToSend = this.batch.tests.splice(0);
-
-    const logData = { action: 'addTestsBatch', tests: testsToSend };
-    if (this.store.runId) logData.runId = this.store.runId;
-    this.logToFile(logData);
+    this.tests.push(data);
+    if (!this.exitListenerAttached) {
+      process.once('exit', this.flushOnExit);
+      this.exitListenerAttached = true;
+    }
   }
 
   async finishRun(params) {
     if (!this.isEnabled) return;
     await this.sync();
-    if (this.batch.intervalFunction) clearInterval(this.batch.intervalFunction);
     const logData = { action: 'finishRun', params };
     if (this.store.runId) logData.runId = this.store.runId;
     this.logToFile(logData);
@@ -133,8 +112,28 @@ export class DebugPipe {
   }
 
   async sync() {
-    if (!this.isEnabled) return;
-    await this.batchUpload();
+    this.flushBufferedTests();
+  }
+
+  /**
+   * Writes any buffered tests to the debug file as a single batch.
+   * Runs synchronously so it can also be invoked from a process `exit` handler,
+   * which is the only chance to persist tests when a hook failure (e.g. a failing
+   * AfterSuite) prevents `finishRun` from being reached. Idempotent: the buffer is
+   * drained on flush, so a later `finishRun`/exit flush is a no-op.
+   */
+  flushBufferedTests() {
+    if (!this.isEnabled || !this.tests.length) return;
+
+    const tests = this.tests.splice(0);
+    const logData = { action: 'addTestsBatch', tests };
+    if (this.store.runId) logData.runId = this.store.runId;
+    this.logToFile(logData);
+
+    if (this.exitListenerAttached) {
+      process.removeListener('exit', this.flushOnExit);
+      this.exitListenerAttached = false;
+    }
   }
 
   toString() {

package/src/pipe/testomatio.js

@@ -5,6 +5,7 @@ import JsonCycle from 'json-cycle';
 import {
   APP_PREFIX,
   STATUS,
+  BATCH_MODE,
   REQUEST_TIMEOUT,
   getCreateRunRequestTimeout,
   REPORTER_REQUEST_RETRIES,
@@ -46,17 +47,25 @@ function parseCiParams(raw) {
 /**
  * @typedef {import('../../types/types.js').Pipe} Pipe
  * @typedef {import('../../types/types.js').TestData} TestData
+ * @typedef {import('../../types/types.js').BatchMode} BatchMode
+ * @typedef {import('../../types/types.js').CreateRunParams} CreateRunParams
  * @class TestomatioPipe
  * @implements {Pipe}
  */
 class TestomatioPipe {
   constructor(params, store) {
     this.batch = {
-      isEnabled: params?.isBatchEnabled ?? !process.env.TESTOMATIO_DISABLE_BATCH_UPLOAD,
+      /** @type {BatchMode}
+       * Batch upload mode:
+       * - `auto`: upload tests automatically by time interval (e.g. every 5 seconds).
+       * - `manual`: buffer tests and upload only when `sync()` is invoked manually.
+       * - `disabled`: send one test per request, no batching.
+       */
+      mode: params.batchMode || (process.env.TESTOMATIO_DISABLE_BATCH_UPLOAD ? BATCH_MODE.DISABLED : BATCH_MODE.AUTO),
       intervalFunction: null, // will be created in createRun by setInterval function
-      intervalTime: 5000, // how often tests are sent
+      intervalTime: 6000, // how often tests are sent
       tests: [], // array of tests in batch
-      batchIndex: 0, // represents the current batch index (starts from 1 and increments by 1 for each batch)
+      batchIndex: 0,  // represents the current batch index (starts from 1 and increments by 1 for each batch)
       numberOfTimesCalledWithoutTests: 0, // how many times batch was called without tests
     };
     this.retriesTimestamps = [];
@@ -222,13 +231,13 @@ class TestomatioPipe {
 
   /**
    * Creates a new run on Testomat.io
-   * @param {{isBatchEnabled?: boolean, kind?: string, configuration?: Record<string, any>}} params
+   * @param {CreateRunParams} params
    * @returns Promise<void>
    */
   async createRun(params = {}) {
-    this.batch.isEnabled = params.isBatchEnabled ?? this.batch.isEnabled;
+    if (params.batchMode) this.batch.mode = params.batchMode;
     if (!this.isEnabled) return;
-    if (this.batch.isEnabled && this.isEnabled)
+    if (this.batch.mode === BATCH_MODE.AUTO && this.isEnabled)
       this.batch.intervalFunction = setInterval(this.#batchUpload, this.batch.intervalTime);
     if (this.store) {
       this.store.runKind = params.kind;
@@ -433,14 +442,14 @@ class TestomatioPipe {
    * Uploads tests as a batch (multiple tests at once). Intended to be used with a setInterval
    */
   #batchUpload = async () => {
-    if (!this.batch.isEnabled) return;
+    if (this.batch.mode === BATCH_MODE.DISABLED) return;
     if (!this.batch.tests.length) return;
     if (this.#cancelTestReportingInCaseOfTooManyReqFailures()) return;
     // prevent infinite loop
     if (this.batch.numberOfTimesCalledWithoutTests > 10) {
       debug('📨 Batch upload: no tests to send for 10 times, stopping batch');
       clearInterval(this.batch.intervalFunction);
-      this.batch.isEnabled = false;
+      this.batch.mode = BATCH_MODE.DISABLED;
     }
     if (!this.batch.tests.length) {
       debug('📨 Batch upload: no tests to send');
@@ -496,11 +505,11 @@ class TestomatioPipe {
     this.#formatData(data);
 
     let uploading = null;
-    if (!this.batch.isEnabled) uploading = this.#uploadSingleTest(data);
+    if (this.batch.mode === BATCH_MODE.DISABLED) uploading = this.#uploadSingleTest(data);
     else this.batch.tests.push(data);
 
-    // if test is added after run which is already finished
-    if (!this.batch.intervalFunction) uploading = this.#batchUpload();
+    // auto mode but no interval running yet (e.g. createRun hasn't started it): flush immediately
+    if (this.batch.mode === BATCH_MODE.AUTO && !this.batch.intervalFunction) uploading = this.#batchUpload();
 
     // return promise to be able to wait for it
     return uploading;
@@ -529,7 +538,7 @@ class TestomatioPipe {
       // (e.g. if test has artifacts, add test function will be invoked only after artifacts are uploaded)
       // batch stops working after run is finished; thus, disable it to use single test uploading
       this.batch.intervalFunction = null;
-      this.batch.isEnabled = false;
+      this.batch.mode = BATCH_MODE.DISABLED;
     }
 
     debug('Finishing run...');
@@ -613,7 +622,7 @@ class TestomatioPipe {
     if (this.batch.intervalFunction) {
       clearInterval(this.batch.intervalFunction);
       this.batch.intervalFunction = null;
-      this.batch.isEnabled = false;
+      this.batch.mode = BATCH_MODE.DISABLED;
     }
     this.batch.tests = [];
   }

package/src/replay.js

@@ -1,7 +1,7 @@
 import fs from 'fs';
 import path from 'path';
 import TestomatClient from './client.js';
-import { STATUS, DEBUG_FILE } from './constants.js';
+import { STATUS, DEBUG_FILE, BATCH_MODE } from './constants.js';
 import { config } from './config.js';
 
 export class Replay {
@@ -216,7 +216,7 @@ export class Replay {
 
     const client = new TestomatClient({
       apiKey: this.apiKey,
-      isBatchEnabled: true,
+      batchMode: BATCH_MODE.AUTO,
       ...runParams,
       ...(runId && { runId }),
     });

package/src/xmlReader.js

@@ -3,7 +3,7 @@ import path from 'path';
 import pc from 'picocolors';
 import fs from 'fs';
 import { XMLParser } from 'fast-xml-parser';
-import { APP_PREFIX, STATUS } from './constants.js';
+import { APP_PREFIX, STATUS, BATCH_MODE } from './constants.js';
 import { randomUUID } from 'crypto';
 import { fileURLToPath } from 'url';
 import { NUnitXmlParser } from './junit-adapter/nunit-parser.js';
@@ -73,8 +73,7 @@ class XmlReader {
       env: TESTOMATIO_ENV,
       group_title: TESTOMATIO_RUNGROUP_TITLE,
       detach: TESTOMATIO_MARK_DETACHED,
-      // batch uploading is implemented for xml already
-      isBatchEnabled: false,
+      batchMode: BATCH_MODE.MANUAL,
     };
     this.runId = opts.runId || TESTOMATIO_RUN;
     this.adapter = adapterFactory(opts.lang?.toLowerCase(), opts);
@@ -543,7 +542,7 @@ class XmlReader {
       title: this.requestParams.title,
       env: this.requestParams.env,
       group_title: this.requestParams.group_title,
-      isBatchEnabled: this.requestParams.isBatchEnabled,
+      batchMode: this.requestParams.batchMode,
     };
 
     debug('Run', runParams);
@@ -611,7 +610,7 @@ class XmlReader {
     this.pipes = this.pipes || (await this.pipesPromise);
 
     // Create run before uploading tests to ensure runId is set
-    await this.createRun();
+    // await this.createRun(); // makes reporting stuck after finish, thus commenting out
 
     if (!this.tests || !Array.isArray(this.tests) || this.tests.length === 0) {
       debug('No tests to upload, finishing run');

package/types/types.d.ts

@@ -234,7 +234,7 @@ export interface HtmlTestData extends TestData {
 /**
  * Extended test data for Markdown reporter.
  */
-export interface MarkdownTestData extends HtmlTestData {}
+export interface MarkdownTestData extends HtmlTestData { }
 
 /**
  * Object representing a result of a Run.
@@ -288,6 +288,13 @@ export enum RunStatus {
   Finished = 'finished',
 }
 
+/** Batch upload strategy:
+ * `auto` (by time interval, e.g. every 5 seconds),
+ * `manual` (send tests via manually invoking sync() ),
+ * `disabled` (one test per request, no batching).
+ */
+export type BatchMode = 'auto' | 'manual' | 'disabled';
+
 export interface Pipe {
   isEnabled: boolean;
   store: {};
@@ -334,8 +341,8 @@ export interface CreateRunParams {
   /** Run configuration merged into the server-side run configuration. */
   configuration?: Record<string, any>;
 
-  /** Override batch upload on/off. */
-  isBatchEnabled?: boolean;
+  /** Override batch upload mode. */
+  batchMode?: BatchMode;
 }
 
 /**