testomatio-reporter-cli 2.21.0 → 2.22.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testomatio-reporter-cli",
-  "version": "2.21.0",
+  "version": "2.22.0",
   "description": "Yarn Berry compatible standalone CLI for @testomatio/reporter",
   "type": "module",
   "bin": {

package/src/bin/cli.js

@@ -28,7 +28,6 @@ import {
   downloadArtifacts,
   defaultArtifactsDir,
   TESTRUNS_DEFAULT_LIMIT,
-  TESTRUNS_MAX_LIMIT,
   TESTRUN_STATUSES,
 } from '../fetch/testruns.js';
 import { renderRunReport } from '../fetch/report.js';
@@ -149,7 +148,7 @@ const fetchCommand = program
   .command('fetch')
   .description('Fetch runs from Testomat.io API v2')
   .argument('[url]', 'URL of a run or of a (filtered) runs page in Testomat.io')
-  .option('--project <slug>', 'Project slug (or set TESTOMATIO_PROJECT)')
+  .option('--project <slug>', 'Project slug for listing runs (or set TESTOMATIO_PROJECT)')
   .option('--run <id>', 'Fetch a single run by its id')
   .option('--title <text>', 'Filter by run title')
   .option('--tql <query>', 'Filter using Testomat Query Language')
@@ -180,29 +179,33 @@ const fetchCommand = program
       }
     }
 
-    const project = opts.project || page.project || process.env.TESTOMATIO_PROJECT;
-    if (!project) {
-      log.error(pc.red('⚠️  --project (or TESTOMATIO_PROJECT) is required'));
-      process.exit(1);
-    }
-
     const baseUrl = page.baseUrl || process.env.TESTOMATIO_URL || 'https://app.testomat.io';
     const runId = opts.run || page.runId;
-    let url;
 
     if (runId) {
-      url = new URL(`/api/v2/${project}/runs/${runId}`, baseUrl);
-    } else {
-      url = new URL(`/api/v2/${project}/runs`, baseUrl);
-      for (const [key, value] of page.params || []) url.searchParams.append(key, value);
-      if (opts.title) url.searchParams.set('search', opts.title);
-      if (opts.tql) url.searchParams.set('tql', opts.tql);
-      if (opts.rungroup) url.searchParams.set('groupId', opts.rungroup);
-      const requestedLimit = opts.latest ? 1 : parseInt(opts.limit, 10) || FETCH_RUNS_DEFAULT_LIMIT;
-      const limit = Math.min(Math.max(requestedLimit, 1), FETCH_RUNS_MAX_LIMIT);
-      url.searchParams.set('per_page', String(limit));
+      const fetcher = new RunFetcher({ baseUrl, project: page.project, runId, apiKey });
+      const run = await fetcher.fetchRun().catch(exitOnFetchError);
+      console.log(formatFetchRunsOutput({ data: run }, opts.format));
+      return;
+    }
+
+    const project = opts.project || page.project || process.env.TESTOMATIO_PROJECT;
+    if (!project) {
+      log.error(
+        pc.red('⚠️  --project (or TESTOMATIO_PROJECT) is required to list runs, unless --run or a URL is passed'),
+      );
+      process.exit(1);
     }
 
+    const url = new URL(`/api/v2/${project}/runs`, baseUrl);
+    for (const [key, value] of page.params || []) url.searchParams.append(key, value);
+    if (opts.title) url.searchParams.set('search', opts.title);
+    if (opts.tql) url.searchParams.set('tql', opts.tql);
+    if (opts.rungroup) url.searchParams.set('groupId', opts.rungroup);
+    const requestedLimit = opts.latest ? 1 : parseInt(opts.limit, 10) || FETCH_RUNS_DEFAULT_LIMIT;
+    const limit = Math.min(Math.max(requestedLimit, 1), FETCH_RUNS_MAX_LIMIT);
+    url.searchParams.set('per_page', String(limit));
+
     const response = await fetch(url, { headers: { Authorization: `Bearer ${apiKey}` } });
     const body = await response.json();
 
@@ -214,17 +217,16 @@ const fetchCommand = program
     console.log(formatFetchRunsOutput(body, opts.format));
   });
 
-// fetch parses the options it shares with its subcommands (--project, --format, --limit) wherever they are placed,
+// fetch parses the options it shares with its subcommands (--format, --limit) wherever they are placed,
 // so subcommands declare them only for help and read them with optsWithGlobals()
 
 /**
  * Build a RunFetcher for the run argument of fetch status|count|report, or exit with an error.
  *
  * @param {string} runArg
- * @param {{project?: string}} opts
  * @returns {RunFetcher}
  */
-function createRunFetcher(runArg, opts) {
+function createRunFetcher(runArg) {
   const apiKey = process.env['INPUT_TESTOMATIO-KEY'] || config.TESTOMATIO;
   if (!apiKey) {
     log.error(pc.red('⚠️  TESTOMATIO API key required'));
@@ -232,7 +234,7 @@ function createRunFetcher(runArg, opts) {
   }
 
   try {
-    return new RunFetcher({ ...resolveRunTarget(runArg, opts), apiKey });
+    return new RunFetcher({ ...resolveRunTarget(runArg), apiKey });
   } catch (err) {
     log.error(pc.red(`⚠️  ${err.message}`));
     process.exit(1);
@@ -268,11 +270,10 @@ fetchCommand
   .command('status')
   .description('Show a run with the number of its testruns by status')
   .argument('<run>', RUN_ARGUMENT_DESCRIPTION)
-  .option('--project <slug>', 'Project slug (or set TESTOMATIO_PROJECT)')
   .option('--format <format>', 'Machine-readable output: run status and counts as JSON (--format json)')
   .action(async (runArg, _opts, command) => {
     const opts = command.optsWithGlobals();
-    const fetcher = createRunFetcher(runArg, opts);
+    const fetcher = createRunFetcher(runArg);
 
     const [run, summary] = await Promise.all([fetcher.fetchRun(), fetcher.countTestrunsByStatus()]).catch(
       exitOnFetchError,
@@ -294,12 +295,11 @@ fetchCommand
   .command('count')
   .description('Print the number of testruns of a run, to pick --limit and filters for fetch report')
   .argument('<run>', RUN_ARGUMENT_DESCRIPTION)
-  .option('--project <slug>', 'Project slug (or set TESTOMATIO_PROJECT)')
   .option('--only <statuses>', STATUS_FILTER_DESCRIPTION)
   .option('--only-failed', 'Only failed testruns (shorthand for --only failed)')
   .action(async (runArg, _opts, command) => {
     const opts = command.optsWithGlobals();
-    const fetcher = createRunFetcher(runArg, opts);
+    const fetcher = createRunFetcher(runArg);
     const statuses = getStatusFilter(opts);
 
     const total = await fetcher.countTestruns(statuses).catch(exitOnFetchError);
@@ -310,22 +310,19 @@ fetchCommand
   .command('report')
   .description('Print a run with its testruns as a Markdown report')
   .argument('<run>', RUN_ARGUMENT_DESCRIPTION)
-  .option('--project <slug>', 'Project slug (or set TESTOMATIO_PROJECT)')
   .option('--only <statuses>', STATUS_FILTER_DESCRIPTION)
   .option('--only-failed', 'Only failed testruns (shorthand for --only failed)')
   .option('--stack', 'Include stack traces')
   .option('--artifacts', 'Download artifacts to --out and link them from the report')
   .option('--out <dir>', 'Directory for downloaded artifacts (default: ./<run-title>-<run-id>)')
-  .option(
-    '--limit <number>',
-    `Max number of testruns, failed first (default ${TESTRUNS_DEFAULT_LIMIT}, max ${TESTRUNS_MAX_LIMIT})`,
-  )
+  .option('--limit <number>', `Max number of testruns, failed first (default ${TESTRUNS_DEFAULT_LIMIT})`)
+  .option('--all', 'All matching testruns, fetched in chunks of 100 (overrides --limit)')
+  .option('--no-defects', 'Skip defects of the run and of its tests')
   .action(async (runArg, _opts, command) => {
     const opts = command.optsWithGlobals();
-    const fetcher = createRunFetcher(runArg, opts);
+    const fetcher = createRunFetcher(runArg);
     const statuses = getStatusFilter(opts);
-    const { limit, capped } = parseTestrunsLimit(opts.limit);
-    if (capped) log.warn(pc.yellow(`⚠️  --limit is capped at ${TESTRUNS_MAX_LIMIT}`));
+    const limit = parseTestrunsLimit(opts);
 
     const run = await fetcher.fetchRun().catch(exitOnFetchError);
 
@@ -337,16 +334,25 @@ fetchCommand
       }
     }
 
-    const [summary, list] = await Promise.all([
+    let runDefectsRequest;
+    if (opts.defects) {
+      runDefectsRequest = fetcher.listRunDefects().catch(err => {
+        log.warn(pc.yellow(`⚠️  Failed to fetch defects of the run: ${err.message}`));
+        return null;
+      });
+    }
+
+    const [summary, list, runDefects] = await Promise.all([
       fetcher.countTestrunsByStatus(),
       fetcher.listTestruns({ statuses, limit }),
+      runDefectsRequest,
     ]).catch(exitOnFetchError);
 
     let { testruns } = list;
     if (list.total > testruns.length) {
       log.warn(
         pc.yellow(
-          `⚠️  Showing ${testruns.length} of ${list.total} testruns. Use --limit or --only-failed to narrow it.`,
+          `⚠️  Showing ${testruns.length} of ${list.total} testruns. Use --all, --limit or --only-failed.`,
         ),
       );
     }
@@ -358,6 +364,14 @@ fetchCommand
       if (!opts.stack) testruns = testruns.map(({ stack, ...testrun }) => testrun);
     }
 
+    if (opts.defects) {
+      testruns = await fetcher.addTestDefects(testruns, (testId, err) => {
+        log.warn(pc.yellow(`⚠️  Failed to fetch defects of test ${testId}: ${err.message}`));
+      });
+    } else {
+      testruns = testruns.map(({ defects, ...testrun }) => testrun);
+    }
+
     if (opts.artifacts) {
       const outDir = opts.out || defaultArtifactsDir(run, fetcher.runId);
       const artifactsCount = testruns.reduce((sum, testrun) => sum + (testrun.artifacts?.length || 0), 0);
@@ -370,7 +384,7 @@ fetchCommand
     }
 
     process.stdout.write(
-      renderRunReport({ run, runUrl: fetcher.runUrl, summary, testruns, matched: list.total, statuses }),
+      renderRunReport({ run, runUrl: fetcher.runUrl, summary, testruns, matched: list.total, statuses, runDefects }),
     );
   });
 

package/src/fetch/report.js

@@ -1,4 +1,4 @@
-import { renderTests } from '../pipe/markdown.js';
+import { renderTests, defectLink } from '../pipe/markdown.js';
 
 const TITLE_SEPARATOR = ' ⇒ ';
 
@@ -28,9 +28,27 @@ function toReportTest(testrun) {
     message: testrun.message,
     stack: testrun.stack,
     artifacts: testrun.artifacts,
+    defects: testrun.defects,
   };
 }
 
+/**
+ * @param {{defects: object[], total: number}} runDefects - Defects of GET /api/v2/:project/defects?run_id=
+ * @returns {string}
+ */
+function renderRunDefects({ defects, total }) {
+  const lines = ['## Defects', ''];
+  if (total > defects.length) lines.push(`${defects.length} of ${total} shown`, '');
+  lines.push('| Defect | Title | Status | Severity |', '| ------ | ----- | ------ | -------- |');
+  for (const defect of defects) {
+    const cells = [defectLink(defect), defect.title, defect.status, defect.severity].map(cell =>
+      String(cell || '').replace(/\|/g, '\\|'),
+    );
+    lines.push(`| ${cells.join(' | ')} |`);
+  }
+  return lines.join('\n');
+}
+
 /**
  * Render a fetched run with its testruns as a Markdown report.
  *
@@ -41,9 +59,10 @@ function toReportTest(testrun) {
  * @param {object[]} params.testruns
  * @param {number} params.matched - Number of testruns matching the filter
  * @param {string[]} params.statuses - Status filter, empty for all testruns
+ * @param {{defects: object[], total: number} | null} [params.runDefects] - Defects of the run
  * @returns {string}
  */
-function renderRunReport({ run, runUrl, summary, testruns, matched, statuses }) {
+function renderRunReport({ run, runUrl, summary, testruns, matched, statuses, runDefects }) {
   const status = String(run.status || 'unknown');
   const sections = [`# ${run.title || run.id} — ${status.toUpperCase()}`];
 
@@ -78,6 +97,8 @@ function renderRunReport({ run, runUrl, summary, testruns, matched, statuses })
     ].join('\n'),
   );
 
+  if (runDefects?.defects.length) sections.push(renderRunDefects(runDefects));
+
   sections.push(renderTests(testruns.map(toReportTest)));
 
   return `${sections.join('\n\n')}\n`;

package/src/fetch/testruns.js

@@ -6,29 +6,26 @@ import { S3Uploader } from '../uploader.js';
 const debug = createDebugMessages('@testomatio/reporter:fetch');
 
 const TESTRUNS_DEFAULT_LIMIT = 50;
-const TESTRUNS_MAX_LIMIT = 100;
+const TESTRUNS_PAGE_SIZE = 100;
 const TESTRUN_STATUSES = ['passed', 'failed', 'skipped', 'pending'];
 const PARALLEL_REQUESTS = 5;
+const FINISHED_RUN_STATUSES = ['passed', 'failed', 'terminated', 'finished'];
 
 /**
  * Resolve the run argument of `fetch status|count|report`: a run id or a run URL.
- * A URL also sets the server and the project.
+ * A URL also sets the server and the project; for a run id the project is resolved from the run by RunFetcher.
  *
  * @param {string} runArg
- * @param {{project?: string}} opts
- * @returns {{baseUrl: string, project: string, runId: string}}
+ * @returns {{baseUrl: string, project?: string, runId: string}}
  */
-function resolveRunTarget(runArg, opts) {
+function resolveRunTarget(runArg) {
   if (/^https?:\/\//i.test(runArg)) {
     const page = parseRunsUrl(runArg);
     if (!page?.runId) throw new Error(`Not a Testomat.io run URL: ${runArg}`);
-    return { baseUrl: page.baseUrl, project: opts.project || page.project, runId: page.runId };
+    return { baseUrl: page.baseUrl, project: page.project, runId: page.runId };
   }
 
-  const project = opts.project || process.env.TESTOMATIO_PROJECT;
-  if (!project) throw new Error('--project (or TESTOMATIO_PROJECT) is required');
-
-  return { baseUrl: process.env.TESTOMATIO_URL || 'https://app.testomat.io', project, runId: runArg };
+  return { baseUrl: process.env.TESTOMATIO_URL || 'https://app.testomat.io', runId: runArg };
 }
 
 /**
@@ -53,12 +50,12 @@ function parseStatuses(opts) {
 }
 
 /**
- * @param {string} [value] - Value of the CLI --limit option.
- * @returns {{limit: number, capped: boolean}}
+ * @param {{limit?: string, all?: boolean}} opts - Values of the CLI --limit and --all options.
+ * @returns {number} Infinity for --all
  */
-function parseTestrunsLimit(value) {
-  const requested = Math.max(parseInt(value, 10) || TESTRUNS_DEFAULT_LIMIT, 1);
-  return { limit: Math.min(requested, TESTRUNS_MAX_LIMIT), capped: requested > TESTRUNS_MAX_LIMIT };
+function parseTestrunsLimit(opts) {
+  if (opts.all) return Infinity;
+  return Math.max(parseInt(opts.limit, 10) || TESTRUNS_DEFAULT_LIMIT, 1);
 }
 
 /**
@@ -88,13 +85,45 @@ async function mapWithLimit(items, limit, fn) {
  */
 class RunFetcher {
   /**
-   * @param {{baseUrl: string, project: string, runId: string, apiKey: string}} params
+   * @param {{baseUrl: string, project?: string, runId: string, apiKey: string}} params
    */
   constructor({ baseUrl, project, runId, apiKey }) {
     this.baseUrl = baseUrl;
     this.project = project;
     this.runId = runId;
     this.apiKey = apiKey;
+    /** @type {Promise<any>|null} */
+    this.reporterRun = null;
+  }
+
+  /**
+   * Empty update of the run in the reporter API: it returns the project of the token and shared S3 credentials.
+   *
+   * @returns {Promise<{project?: {id: string}, artifacts?: object}>}
+   */
+  updateReporterRun() {
+    this.reporterRun ||= (async () => {
+      const url = new URL(`/api/reporter/${this.runId}`, this.baseUrl);
+      debug('PUT', url.toString());
+      const response = await fetch(url, {
+        method: 'PUT',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ api_key: this.apiKey }),
+      });
+      const body = await response.json().catch(() => ({}));
+      if (response.status === 404) throw new Error(`Run ${this.runId} not found`);
+      if (!response.ok) throw new Error(`${response.status} ${body.message || body.error || response.statusText}`);
+      return body;
+    })();
+    return this.reporterRun;
+  }
+
+  async resolveProject() {
+    if (this.project) return this.project;
+    const body = await this.updateReporterRun();
+    if (!body.project?.id) throw new Error(`Project of run ${this.runId} is unknown`);
+    this.project = body.project.id;
+    return this.project;
   }
 
   get runUrl() {
@@ -102,6 +131,7 @@ class RunFetcher {
   }
 
   async get(apiPath, params = []) {
+    await this.resolveProject();
     const url = new URL(`/api/v2/${this.project}${apiPath}`, this.baseUrl);
     for (const [key, value] of params) url.searchParams.append(key, value);
     debug('GET', url.toString());
@@ -148,19 +178,31 @@ class RunFetcher {
   }
 
   /**
-   * Testruns with failures first; the list has no stack and no artifacts.
+   * Testruns with failures first, read in pages of TESTRUNS_PAGE_SIZE; the list has no stack and no artifacts.
    *
-   * @param {{statuses?: string[], limit: number}} params
+   * @param {{statuses?: string[], limit: number}} params - limit is Infinity to read all testruns
    * @returns {Promise<{testruns: object[], total: number}>}
    */
   async listTestruns({ statuses = [], limit }) {
-    const body = await this.get('/testruns', [
-      ...this.testrunParams(statuses),
-      ['sort', 'failure'],
-      ['order', 'desc'],
-      ['per_page', String(limit)],
-    ]);
-    return { testruns: body.data || [], total: body.meta?.total || 0 };
+    const perPage = Math.min(limit, TESTRUNS_PAGE_SIZE);
+    const getPage = async page => {
+      const body = await this.get('/testruns', [
+        ...this.testrunParams(statuses),
+        ['sort', 'failure'],
+        ['order', 'desc'],
+        ['per_page', String(perPage)],
+        ['page', String(page)],
+      ]);
+      return { testruns: body.data || [], total: body.meta?.total || 0 };
+    };
+
+    const first = await getPage(1);
+    const pagesCount = Math.ceil(Math.min(limit, first.total) / perPage);
+    const nextPages = Array.from({ length: Math.max(pagesCount - 1, 0) }, (_, index) => index + 2);
+    const rest = await mapWithLimit(nextPages, PARALLEL_REQUESTS, getPage);
+
+    const testruns = [first, ...rest].flatMap(page => page.testruns).slice(0, limit);
+    return { testruns, total: first.total };
   }
 
   /**
@@ -181,31 +223,70 @@ class RunFetcher {
     });
   }
 
+  /**
+   * Defects linked to the testruns of the run; one page of TESTRUNS_PAGE_SIZE.
+   *
+   * @returns {Promise<{defects: object[], total: number}>}
+   */
+  async listRunDefects() {
+    const body = await this.get('/defects', [
+      ['run_id', this.runId],
+      ['per_page', String(TESTRUNS_PAGE_SIZE)],
+    ]);
+    return { defects: body.data || [], total: body.meta?.total || 0 };
+  }
+
+  /**
+   * Add the defects linked to the test case of each testrun to the testrun's own defects.
+   * A test that fails to load keeps only the testrun defects.
+   *
+   * @param {object[]} testruns
+   * @param {(testId: string, err: Error) => void} onError
+   */
+  async addTestDefects(testruns, onError) {
+    const testIds = [...new Set(testruns.map(testrun => testrun.testId).filter(Boolean))];
+    const testDefects = await mapWithLimit(testIds, PARALLEL_REQUESTS, async testId => {
+      try {
+        const body = await this.get('/defects', [
+          ['test_id', testId],
+          ['per_page', String(TESTRUNS_PAGE_SIZE)],
+        ]);
+        return body.data || [];
+      } catch (err) {
+        onError(testId, err);
+        return [];
+      }
+    });
+    const defectsByTest = new Map(testIds.map((testId, index) => [testId, testDefects[index]]));
+
+    return testruns.map(testrun => {
+      const defects = [...(testrun.defects || []), ...(defectsByTest.get(testrun.testId) || [])];
+      const unique = [...new Map(defects.map(defect => [defect.id, defect])).values()];
+      return { ...testrun, defects: unique };
+    });
+  }
+
   /**
    * Shared S3 credentials of the project, returned by an empty update of the run.
-   * Skipped for scheduled runs (the update launches them) and mixed runs (it marks automated tests as started).
+   * Unless the run was already updated to resolve its project, skipped for scheduled runs (the update launches them)
+   * and unfinished mixed runs (it marks automated tests as started).
    *
    * @param {{status?: string, kind?: string}} run
    * @returns {Promise<boolean>} whether credentials were applied
    */
   async applySharedS3Credentials(run) {
-    if (run.status === 'scheduled' || run.kind === 'mixed') {
+    const isUnfinishedMixed = run.kind === 'mixed' && !FINISHED_RUN_STATUSES.includes(run.status);
+    if (!this.reporterRun && (run.status === 'scheduled' || isUnfinishedMixed)) {
       debug(`Shared S3 credentials are not requested for a ${run.status} ${run.kind} run`);
       return false;
     }
 
-    const url = new URL(`/api/reporter/${this.runId}`, this.baseUrl);
-    const response = await fetch(url, {
-      method: 'PUT',
-      headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({ api_key: this.apiKey }),
+    const body = await this.updateReporterRun().catch(err => {
+      debug('Failed to get shared S3 credentials:', err.message);
+      return { artifacts: null };
     });
-    if (!response.ok) {
-      debug('Failed to get shared S3 credentials:', response.status);
-      return false;
-    }
 
-    const { artifacts } = await response.json();
+    const { artifacts } = body;
     if (!artifacts || !Object.keys(artifacts).length) return false;
     setS3Credentials(artifacts);
     return true;
@@ -271,7 +352,7 @@ function defaultArtifactsDir(run, runId) {
 
 export {
   TESTRUNS_DEFAULT_LIMIT,
-  TESTRUNS_MAX_LIMIT,
+  TESTRUNS_PAGE_SIZE,
   TESTRUN_STATUSES,
   RunFetcher,
   resolveRunTarget,

package/src/pipe/markdown.js

@@ -332,6 +332,9 @@ function renderTest(test) {
   const logsBlock = renderLogs(test);
   if (logsBlock) lines.push(logsBlock);
 
+  const defectsBlock = renderDefects(test);
+  if (defectsBlock) lines.push(defectsBlock);
+
   const artifactsBlock = renderArtifacts(test);
   if (artifactsBlock) lines.push(artifactsBlock);
 
@@ -497,6 +500,26 @@ function renderArtifacts(test) {
   return lines.join('\n');
 }
 
+function renderDefects(test) {
+  if (!Array.isArray(test.defects) || !test.defects.length) return '';
+
+  const lines = ['**Defects**', ''];
+  for (const defect of test.defects) {
+    let line = `- ${defectLink(defect)}`;
+    if (defect.sourceKey && defect.title) line += ` ${mdInline(defect.title)}`;
+    const details = [defect.status, defect.severity].filter(Boolean);
+    if (details.length) line += ` (${details.join(', ')})`;
+    lines.push(line);
+  }
+  return lines.join('\n');
+}
+
+function defectLink(defect) {
+  const key = mdInline(defect.sourceKey || defect.title || 'defect');
+  if (!defect.url) return key;
+  return `[${key}](${defect.url})`;
+}
+
 function normalizeArtifact(raw) {
   if (raw == null) return null;
 
@@ -755,6 +778,6 @@ function aggregateTestRetries(tests) {
   return aggregated;
 }
 
-export { renderTests };
+export { renderTests, defectLink };
 
 export default MarkdownPipe;