testomatio-reporter-cli 2.20.0 → 2.22.0

package/src/fetch/report.js

@@ -0,0 +1,107 @@
+import { renderTests, defectLink } from '../pipe/markdown.js';
+
+const TITLE_SEPARATOR = ' ⇒ ';
+
+/**
+ * Map a testrun of API v2 to the test shape of the Markdown pipe.
+ * Without a suite, the API returns the title as "<suite> ⇒ <title>".
+ *
+ * @param {object} testrun
+ * @returns {object}
+ */
+function toReportTest(testrun) {
+  let title = String(testrun.test_title || '').trim();
+  let suite = testrun.suiteTitle;
+
+  if (!suite && title.includes(TITLE_SEPARATOR)) {
+    const [suitePart, ...titleParts] = title.split(TITLE_SEPARATOR);
+    suite = suitePart.trim();
+    title = titleParts.join(TITLE_SEPARATOR).trim();
+  }
+
+  return {
+    title,
+    suite_title: suite,
+    status: testrun.status,
+    run_time: testrun.run_time,
+    test_id: testrun.testId,
+    message: testrun.message,
+    stack: testrun.stack,
+    artifacts: testrun.artifacts,
+    defects: testrun.defects,
+  };
+}
+
+/**
+ * @param {{defects: object[], total: number}} runDefects - Defects of GET /api/v2/:project/defects?run_id=
+ * @returns {string}
+ */
+function renderRunDefects({ defects, total }) {
+  const lines = ['## Defects', ''];
+  if (total > defects.length) lines.push(`${defects.length} of ${total} shown`, '');
+  lines.push('| Defect | Title | Status | Severity |', '| ------ | ----- | ------ | -------- |');
+  for (const defect of defects) {
+    const cells = [defectLink(defect), defect.title, defect.status, defect.severity].map(cell =>
+      String(cell || '').replace(/\|/g, '\\|'),
+    );
+    lines.push(`| ${cells.join(' | ')} |`);
+  }
+  return lines.join('\n');
+}
+
+/**
+ * Render a fetched run with its testruns as a Markdown report.
+ *
+ * @param {object} params
+ * @param {object} params.run - Run from GET /api/v2/:project/runs/:id
+ * @param {string} params.runUrl
+ * @param {{total: number, byStatus: Record<string, number>}} params.summary - Counts of all testruns of the run
+ * @param {object[]} params.testruns
+ * @param {number} params.matched - Number of testruns matching the filter
+ * @param {string[]} params.statuses - Status filter, empty for all testruns
+ * @param {{defects: object[], total: number} | null} [params.runDefects] - Defects of the run
+ * @returns {string}
+ */
+function renderRunReport({ run, runUrl, summary, testruns, matched, statuses, runDefects }) {
+  const status = String(run.status || 'unknown');
+  const sections = [`# ${run.title || run.id} — ${status.toUpperCase()}`];
+
+  const rows = [
+    ['Run ID', `\`${run.id}\``],
+    ['Run URL', `<${runUrl}>`],
+    ['Status', status],
+  ];
+  if (run.env) rows.push(['Environment', run.env]);
+  if (run.launched_at) rows.push(['Started', run.launched_at]);
+  if (run.finished_at) rows.push(['Finished', run.finished_at]);
+
+  let shown = `${testruns.length} of ${matched} shown`;
+  if (statuses.length) shown += ` (only: ${statuses.join(', ')})`;
+  rows.push(['Testruns', shown]);
+
+  sections.push(
+    ['| Key | Value |', '| --- | ----- |', ...rows.map(([key, value]) => `| ${key} | ${value} |`)].join('\n'),
+  );
+
+  if (run.description?.trim()) sections.push(`## Description\n\n${run.description.trim()}`);
+
+  const counts = Object.entries(summary.byStatus).sort(([, a], [, b]) => b - a);
+  sections.push(
+    [
+      '## Summary',
+      '',
+      '| Status | Count |',
+      '| ------ | ----- |',
+      ...counts.map(([name, count]) => `| ${name} | ${count} |`),
+      `| total | ${summary.total} |`,
+    ].join('\n'),
+  );
+
+  if (runDefects?.defects.length) sections.push(renderRunDefects(runDefects));
+
+  sections.push(renderTests(testruns.map(toReportTest)));
+
+  return `${sections.join('\n\n')}\n`;
+}
+
+export { renderRunReport, toReportTest };

package/src/fetch/testruns.js

@@ -0,0 +1,364 @@
+import createDebugMessages from 'debug';
+import path from 'path';
+import { parseRunsUrl, setS3Credentials } from '../utils/pipe_utils.js';
+import { S3Uploader } from '../uploader.js';
+
+const debug = createDebugMessages('@testomatio/reporter:fetch');
+
+const TESTRUNS_DEFAULT_LIMIT = 50;
+const TESTRUNS_PAGE_SIZE = 100;
+const TESTRUN_STATUSES = ['passed', 'failed', 'skipped', 'pending'];
+const PARALLEL_REQUESTS = 5;
+const FINISHED_RUN_STATUSES = ['passed', 'failed', 'terminated', 'finished'];
+
+/**
+ * Resolve the run argument of `fetch status|count|report`: a run id or a run URL.
+ * A URL also sets the server and the project; for a run id the project is resolved from the run by RunFetcher.
+ *
+ * @param {string} runArg
+ * @returns {{baseUrl: string, project?: string, runId: string}}
+ */
+function resolveRunTarget(runArg) {
+  if (/^https?:\/\//i.test(runArg)) {
+    const page = parseRunsUrl(runArg);
+    if (!page?.runId) throw new Error(`Not a Testomat.io run URL: ${runArg}`);
+    return { baseUrl: page.baseUrl, project: page.project, runId: page.runId };
+  }
+
+  return { baseUrl: process.env.TESTOMATIO_URL || 'https://app.testomat.io', runId: runArg };
+}
+
+/**
+ * Collect the statuses passed with --only and --only-failed.
+ *
+ * @param {{only?: string, onlyFailed?: boolean}} opts
+ * @returns {string[]}
+ */
+function parseStatuses(opts) {
+  const statuses = String(opts.only || '')
+    .split(',')
+    .map(status => status.trim().toLowerCase())
+    .filter(Boolean);
+  if (opts.onlyFailed) statuses.push('failed');
+
+  const unknown = statuses.filter(status => !TESTRUN_STATUSES.includes(status));
+  if (unknown.length) {
+    throw new Error(`Unknown status: ${unknown.join(', ')}. Use one of: ${TESTRUN_STATUSES.join(', ')}`);
+  }
+
+  return [...new Set(statuses)];
+}
+
+/**
+ * @param {{limit?: string, all?: boolean}} opts - Values of the CLI --limit and --all options.
+ * @returns {number} Infinity for --all
+ */
+function parseTestrunsLimit(opts) {
+  if (opts.all) return Infinity;
+  return Math.max(parseInt(opts.limit, 10) || TESTRUNS_DEFAULT_LIMIT, 1);
+}
+
+/**
+ * Run `fn` over `items` with at most `limit` calls in flight; results keep the order of `items`.
+ *
+ * @template T, R
+ * @param {T[]} items
+ * @param {number} limit
+ * @param {(item: T) => Promise<R>} fn
+ * @returns {Promise<R[]>}
+ */
+async function mapWithLimit(items, limit, fn) {
+  const results = new Array(items.length);
+  let next = 0;
+  const worker = async () => {
+    while (next < items.length) {
+      const index = next++;
+      results[index] = await fn(items[index]);
+    }
+  };
+  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
+  return results;
+}
+
+/**
+ * Read-only client for a single run and its testruns in API v2.
+ */
+class RunFetcher {
+  /**
+   * @param {{baseUrl: string, project?: string, runId: string, apiKey: string}} params
+   */
+  constructor({ baseUrl, project, runId, apiKey }) {
+    this.baseUrl = baseUrl;
+    this.project = project;
+    this.runId = runId;
+    this.apiKey = apiKey;
+    /** @type {Promise<any>|null} */
+    this.reporterRun = null;
+  }
+
+  /**
+   * Empty update of the run in the reporter API: it returns the project of the token and shared S3 credentials.
+   *
+   * @returns {Promise<{project?: {id: string}, artifacts?: object}>}
+   */
+  updateReporterRun() {
+    this.reporterRun ||= (async () => {
+      const url = new URL(`/api/reporter/${this.runId}`, this.baseUrl);
+      debug('PUT', url.toString());
+      const response = await fetch(url, {
+        method: 'PUT',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ api_key: this.apiKey }),
+      });
+      const body = await response.json().catch(() => ({}));
+      if (response.status === 404) throw new Error(`Run ${this.runId} not found`);
+      if (!response.ok) throw new Error(`${response.status} ${body.message || body.error || response.statusText}`);
+      return body;
+    })();
+    return this.reporterRun;
+  }
+
+  async resolveProject() {
+    if (this.project) return this.project;
+    const body = await this.updateReporterRun();
+    if (!body.project?.id) throw new Error(`Project of run ${this.runId} is unknown`);
+    this.project = body.project.id;
+    return this.project;
+  }
+
+  get runUrl() {
+    return new URL(`/projects/${this.project}/runs/${this.runId}`, this.baseUrl).toString();
+  }
+
+  async get(apiPath, params = []) {
+    await this.resolveProject();
+    const url = new URL(`/api/v2/${this.project}${apiPath}`, this.baseUrl);
+    for (const [key, value] of params) url.searchParams.append(key, value);
+    debug('GET', url.toString());
+
+    const response = await fetch(url, { headers: { Authorization: `Bearer ${this.apiKey}` } });
+    const body = await response.json().catch(() => ({}));
+    if (response.status === 404 && apiPath === `/runs/${this.runId}`) {
+      throw new Error(`Run ${this.runId} not found in project ${this.project}`);
+    }
+    if (!response.ok) throw new Error(`${response.status} ${body.error || response.statusText}`);
+    return body;
+  }
+
+  async fetchRun() {
+    const body = await this.get(`/runs/${this.runId}`);
+    return body.data;
+  }
+
+  /**
+   * @param {string[]} statuses
+   * @returns {string[][]}
+   */
+  testrunParams(statuses = []) {
+    const params = [['run_id', this.runId]];
+    for (const status of statuses) params.push(['filter[status][]', status]);
+    return params;
+  }
+
+  /**
+   * @param {string[]} [statuses]
+   * @returns {Promise<number>}
+   */
+  async countTestruns(statuses = []) {
+    const body = await this.get('/testruns', [...this.testrunParams(statuses), ['count', 'true']]);
+    return body.meta?.total || 0;
+  }
+
+  /**
+   * @returns {Promise<{total: number, byStatus: Record<string, number>}>}
+   */
+  async countTestrunsByStatus() {
+    const body = await this.get('/testruns', [...this.testrunParams(), ['count', 'true'], ['group_by', 'status']]);
+    return { total: body.meta?.total || 0, byStatus: body.meta?.group_by || {} };
+  }
+
+  /**
+   * Testruns with failures first, read in pages of TESTRUNS_PAGE_SIZE; the list has no stack and no artifacts.
+   *
+   * @param {{statuses?: string[], limit: number}} params - limit is Infinity to read all testruns
+   * @returns {Promise<{testruns: object[], total: number}>}
+   */
+  async listTestruns({ statuses = [], limit }) {
+    const perPage = Math.min(limit, TESTRUNS_PAGE_SIZE);
+    const getPage = async page => {
+      const body = await this.get('/testruns', [
+        ...this.testrunParams(statuses),
+        ['sort', 'failure'],
+        ['order', 'desc'],
+        ['per_page', String(perPage)],
+        ['page', String(page)],
+      ]);
+      return { testruns: body.data || [], total: body.meta?.total || 0 };
+    };
+
+    const first = await getPage(1);
+    const pagesCount = Math.ceil(Math.min(limit, first.total) / perPage);
+    const nextPages = Array.from({ length: Math.max(pagesCount - 1, 0) }, (_, index) => index + 2);
+    const rest = await mapWithLimit(nextPages, PARALLEL_REQUESTS, getPage);
+
+    const testruns = [first, ...rest].flatMap(page => page.testruns).slice(0, limit);
+    return { testruns, total: first.total };
+  }
+
+  /**
+   * Add `stack` and `artifacts` to each testrun; a testrun that fails to load is kept as is.
+   *
+   * @param {object[]} testruns
+   * @param {(testrun: object, err: Error) => void} onError
+   */
+  async addTestrunDetails(testruns, onError) {
+    return mapWithLimit(testruns, PARALLEL_REQUESTS, async testrun => {
+      try {
+        const body = await this.get(`/testruns/${testrun.id}`);
+        return { ...testrun, stack: body.data?.stack, artifacts: body.data?.artifacts || [] };
+      } catch (err) {
+        onError(testrun, err);
+        return testrun;
+      }
+    });
+  }
+
+  /**
+   * Defects linked to the testruns of the run; one page of TESTRUNS_PAGE_SIZE.
+   *
+   * @returns {Promise<{defects: object[], total: number}>}
+   */
+  async listRunDefects() {
+    const body = await this.get('/defects', [
+      ['run_id', this.runId],
+      ['per_page', String(TESTRUNS_PAGE_SIZE)],
+    ]);
+    return { defects: body.data || [], total: body.meta?.total || 0 };
+  }
+
+  /**
+   * Add the defects linked to the test case of each testrun to the testrun's own defects.
+   * A test that fails to load keeps only the testrun defects.
+   *
+   * @param {object[]} testruns
+   * @param {(testId: string, err: Error) => void} onError
+   */
+  async addTestDefects(testruns, onError) {
+    const testIds = [...new Set(testruns.map(testrun => testrun.testId).filter(Boolean))];
+    const testDefects = await mapWithLimit(testIds, PARALLEL_REQUESTS, async testId => {
+      try {
+        const body = await this.get('/defects', [
+          ['test_id', testId],
+          ['per_page', String(TESTRUNS_PAGE_SIZE)],
+        ]);
+        return body.data || [];
+      } catch (err) {
+        onError(testId, err);
+        return [];
+      }
+    });
+    const defectsByTest = new Map(testIds.map((testId, index) => [testId, testDefects[index]]));
+
+    return testruns.map(testrun => {
+      const defects = [...(testrun.defects || []), ...(defectsByTest.get(testrun.testId) || [])];
+      const unique = [...new Map(defects.map(defect => [defect.id, defect])).values()];
+      return { ...testrun, defects: unique };
+    });
+  }
+
+  /**
+   * Shared S3 credentials of the project, returned by an empty update of the run.
+   * Unless the run was already updated to resolve its project, skipped for scheduled runs (the update launches them)
+   * and unfinished mixed runs (it marks automated tests as started).
+   *
+   * @param {{status?: string, kind?: string}} run
+   * @returns {Promise<boolean>} whether credentials were applied
+   */
+  async applySharedS3Credentials(run) {
+    const isUnfinishedMixed = run.kind === 'mixed' && !FINISHED_RUN_STATUSES.includes(run.status);
+    if (!this.reporterRun && (run.status === 'scheduled' || isUnfinishedMixed)) {
+      debug(`Shared S3 credentials are not requested for a ${run.status} ${run.kind} run`);
+      return false;
+    }
+
+    const body = await this.updateReporterRun().catch(err => {
+      debug('Failed to get shared S3 credentials:', err.message);
+      return { artifacts: null };
+    });
+
+    const { artifacts } = body;
+    if (!artifacts || !Object.keys(artifacts).length) return false;
+    setS3Credentials(artifacts);
+    return true;
+  }
+}
+
+/**
+ * Download the artifacts of each testrun to `<outDir>/<testrun id>/<file name>`.
+ * Each artifact becomes `{name, href}`, where `href` is the local path relative to the current directory,
+ * or the remote URL when the download failed.
+ *
+ * @param {object[]} testruns
+ * @param {string} outDir
+ * @param {(url: string, err: Error) => void} onError
+ */
+async function downloadArtifacts(testruns, outDir, onError) {
+  const uploader = new S3Uploader();
+  const jobs = [];
+
+  // the file name is parsed inside mapWithLimit, so a malformed artifact URL only skips that artifact
+  for (const testrun of testruns) {
+    const usedNames = new Set();
+    for (const url of testrun.artifacts || []) {
+      jobs.push({ testrun, url, outDir, usedNames });
+    }
+  }
+
+  const downloaded = await mapWithLimit(jobs, PARALLEL_REQUESTS, async job => {
+    let name = 'artifact';
+    try {
+      name = decodeURIComponent(path.basename(new URL(job.url).pathname)) || 'artifact';
+      if (job.usedNames.has(name)) name = `${job.usedNames.size}-${name}`;
+      job.usedNames.add(name);
+      const filePath = path.join(job.outDir, String(job.testrun.id), name);
+      await uploader.downloadFile(job.url, filePath);
+      const href = `./${path.relative(process.cwd(), filePath).split(path.sep).join('/')}`;
+      return { testrun: job.testrun, href, name };
+    } catch (err) {
+      onError(job.url, err);
+      return { testrun: job.testrun, href: job.url, name: `${name} (not downloaded)` };
+    }
+  });
+
+  return testruns.map(testrun => ({
+    ...testrun,
+    artifacts: downloaded.filter(job => job.testrun === testrun).map(({ name, href }) => ({ name, href })),
+  }));
+}
+
+/**
+ * @param {{title?: string}} run
+ * @param {string} runId
+ * @returns {string}
+ */
+function defaultArtifactsDir(run, runId) {
+  const slug = String(run.title || 'run')
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+    .slice(0, 60);
+  return `./${slug || 'run'}-${runId}`;
+}
+
+export {
+  TESTRUNS_DEFAULT_LIMIT,
+  TESTRUNS_PAGE_SIZE,
+  TESTRUN_STATUSES,
+  RunFetcher,
+  resolveRunTarget,
+  parseStatuses,
+  parseTestrunsLimit,
+  mapWithLimit,
+  downloadArtifacts,
+  defaultArtifactsDir,
+};

package/src/pipe/markdown.js

@@ -332,6 +332,9 @@ function renderTest(test) {
   const logsBlock = renderLogs(test);
   if (logsBlock) lines.push(logsBlock);
 
+  const defectsBlock = renderDefects(test);
+  if (defectsBlock) lines.push(defectsBlock);
+
   const artifactsBlock = renderArtifacts(test);
   if (artifactsBlock) lines.push(artifactsBlock);
 
@@ -497,6 +500,26 @@ function renderArtifacts(test) {
   return lines.join('\n');
 }
 
+function renderDefects(test) {
+  if (!Array.isArray(test.defects) || !test.defects.length) return '';
+
+  const lines = ['**Defects**', ''];
+  for (const defect of test.defects) {
+    let line = `- ${defectLink(defect)}`;
+    if (defect.sourceKey && defect.title) line += ` ${mdInline(defect.title)}`;
+    const details = [defect.status, defect.severity].filter(Boolean);
+    if (details.length) line += ` (${details.join(', ')})`;
+    lines.push(line);
+  }
+  return lines.join('\n');
+}
+
+function defectLink(defect) {
+  const key = mdInline(defect.sourceKey || defect.title || 'defect');
+  if (!defect.url) return key;
+  return `[${key}](${defect.url})`;
+}
+
 function normalizeArtifact(raw) {
   if (raw == null) return null;
 
@@ -521,6 +544,11 @@ function normalizeArtifact(raw) {
     return { name: path.basename(abs), href, isImage: looksLikeImage(abs) };
   }
 
+  // already resolved link, e.g. a downloaded artifact relative to the report
+  if (typeof raw.href === 'string' && raw.href) {
+    return { name: raw.name || path.basename(raw.href), href: raw.href, isImage: looksLikeImage(raw.href) };
+  }
+
   const rawPath = raw.path || raw.link || raw.url;
   if (!rawPath || typeof rawPath !== 'string') return null;
 
@@ -750,4 +778,6 @@ function aggregateTestRetries(tests) {
   return aggregated;
 }
 
+export { renderTests, defectLink };
+
 export default MarkdownPipe;

package/src/reporter-functions.js

@@ -1,8 +1,11 @@
+import { createRequire } from 'module';
 import { playwrightLogsMarkers } from './adapter/utils/playwright.js';
 import { isPlaywright } from './helpers.js';
 import { services } from './services/index.js';
 import pc from 'picocolors';
 
+const requireModule = createRequire(`${process.cwd()}/package.json`);
+
 /**
  * Stores path to file as artifact and uploads it to the S3 storage
  * @param {string | {path: string, type: string, name: string}} data - path to file or object with path, type and name
@@ -44,6 +47,36 @@ function addStep(message, logs) {
   else services.logger.step(message, logs);
 }
 
+/**
+ * Returns the currently running Vitest test task, so `meta()` (and similar
+ * functions) can attach data to it. The task `.meta` object is serialized by
+ * Vitest back to the reporter (works across all pools: threads, forks, vmThreads).
+ *
+ * Resolution order matters:
+ * 1. `globalThis.__vitest_worker__.current` — set by Vitest in every worker and
+ *    independent of module resolution, so it works even when the reporter is the
+ *    externalized package and `@vitest/runner` is deduped to a different instance
+ *    (monorepos, pnpm, hoisting). This is also populated inside hooks (afterEach),
+ *    pointing at the test that the hook belongs to.
+ * 2. `@vitest/runner` `getCurrentTest()` — fallback for setups/versions where the
+ *    worker global is not exposed.
+ *
+ * @returns {any | null} Vitest test task or null when no test is active
+ */
+function getCurrentVitestTest() {
+  // @ts-ignore - injected by Vitest into the worker global scope
+  const worker = globalThis.__vitest_worker__;
+  const current = worker?.current;
+  if (current && (current.type === 'test' || current.type === undefined)) return current;
+
+  try {
+    const test = requireModule('@vitest/runner').getCurrentTest?.();
+    if (test) return test;
+  } catch {}
+
+  return null;
+}
+
 /**
  * Add key-value pair(s) to the test report
  * @param {{[key: string]: string} | string} keyValue - object { key: value } (multiple props allowed) OR key (string)
@@ -67,6 +100,14 @@ function setKeyValue(keyValue, value = undefined) {
     return;
   }
 
+  if (process.env.VITEST || process.env.VITEST_WORKER_ID) {
+    const vitestTest = getCurrentVitestTest();
+    if (vitestTest) {
+      Object.assign((vitestTest.meta ||= {}), keyValue);
+      return;
+    }
+  }
+
   // in this case keyValue is expected to be an object
   services.keyValues.put(keyValue);
 }

package/src/uploader.js

@@ -347,6 +347,33 @@ export class S3Uploader {
     );
   }
 
+  /**
+   * Download an artifact by its link: from the configured bucket through S3, otherwise over plain HTTP.
+   *
+   * @param {string} link
+   * @param {string} filePath
+   */
+  async downloadFile(link, filePath) {
+    const { S3_BUCKET } = this.getConfig();
+    const Key = getS3KeyFromUrl(link, S3_BUCKET);
+    let body;
+
+    if (Key) {
+      debug('Downloading from S3:', Key);
+      const s3 = new S3(this.#getS3Config());
+      const object = await s3.getObject({ Bucket: S3_BUCKET, Key });
+      body = Buffer.from(await object.Body.transformToByteArray());
+    } else {
+      debug('Downloading over HTTP:', link);
+      const response = await fetch(link);
+      if (!response.ok) throw new Error(`${response.status} ${response.statusText}`);
+      body = Buffer.from(await response.arrayBuffer());
+    }
+
+    fs.mkdirSync(path.dirname(filePath), { recursive: true });
+    fs.writeFileSync(filePath, body);
+  }
+
   async getS3LocationLink(out) {
     const response = await out.done();
 
@@ -409,3 +436,27 @@ export class S3Uploader {
     return cfg;
   }
 }
+
+/**
+ * Object key of an artifact link in the given bucket, for both virtual-hosted (bucket.host/key)
+ * and path style (host/bucket/key) links; null when the link points to another bucket.
+ *
+ * @param {string} link
+ * @param {string} [bucket]
+ * @returns {string|null}
+ */
+export function getS3KeyFromUrl(link, bucket) {
+  if (!bucket) return null;
+
+  let url;
+  try {
+    url = new URL(link);
+  } catch (e) {
+    return null;
+  }
+
+  const objectPath = decodeURIComponent(url.pathname.slice(1));
+  if (url.hostname.startsWith(`${bucket}.`)) return objectPath || null;
+  if (objectPath.startsWith(`${bucket}/`)) return objectPath.slice(bucket.length + 1) || null;
+  return null;
+}