playwright-slack-report 1.1.22 → 1.1.24

package/README.md

@@ -179,6 +179,8 @@ A function that returns a layout object, this configuration is optional.  See se
 Same as **layout** above, but asynchronous in that it returns a promise.
 ### **maxNumberOfFailuresToShow**
 Limits the number of failures shown in the Slack message, defaults to 10.
+### **separateFlaky**
+Don't consider flaky tests as passed, instead output them separately. Flaky tests are tests that passed but not on the first try.  This defaults to `false` but it is highly recommended that you switch this on.
 ### **slackOAuthToken**
 Instead of providing an environment variable `SLACK_BOT_USER_OAUTH_TOKEN` you can specify the token in the config in the `slackOAuthToken` field.
 ### **slackLogLevel** (default LogLevel.DEBUG)

package/dist/src/LayoutGenerator.js

@@ -2,8 +2,6 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.generateFailures = exports.generateBlocks = void 0;
 const generateBlocks = async (summaryResults, maxNumberOfFailures) => {
-    const maxNumberOfFailureLength = 650;
-    const fails = [];
     const meta = [];
     const header = {
         type: 'section',
@@ -16,35 +14,12 @@ const generateBlocks = async (summaryResults, maxNumberOfFailures) => {
         type: 'section',
         text: {
             type: 'mrkdwn',
-            text: `✅ *${summaryResults.passed}* | ❌ *${summaryResults.failed}* | ⏩ *${summaryResults.skipped}*`,
+            text: `✅ *${summaryResults.passed}* | ❌ *${summaryResults.failed}* |${summaryResults.flaky !== undefined
+                ? ` 🟡 *${summaryResults.flaky}* | `
+                : ' '}⏩ *${summaryResults.skipped}*`,
         },
     };
-    for (let i = 0; i < summaryResults.failures.length; i += 1) {
-        const { failureReason, test } = summaryResults.failures[i];
-        const formattedFailure = failureReason
-            .substring(0, maxNumberOfFailureLength)
-            .split('\n')
-            .map((l) => `>${l}`)
-            .join('\n');
-        fails.push({
-            type: 'section',
-            text: {
-                type: 'mrkdwn',
-                text: `*${test}*
-        \n${formattedFailure}`,
-            },
-        });
-        if (i > maxNumberOfFailures) {
-            fails.push({
-                type: 'section',
-                text: {
-                    type: 'mrkdwn',
-                    text: `*There are too many failures to display - ${fails.length} out of ${summaryResults.failures.length} failures shown*`,
-                },
-            });
-            break;
-        }
-    }
+    const fails = await generateFailures(summaryResults, maxNumberOfFailures);
     if (summaryResults.meta) {
         for (let i = 0; i < summaryResults.meta.length; i += 1) {
             const { key, value } = summaryResults.meta[i];
@@ -57,21 +32,14 @@ const generateBlocks = async (summaryResults, maxNumberOfFailures) => {
             });
         }
     }
-    return [
-        header,
-        summary,
-        ...meta,
-        {
-            type: 'divider',
-        },
-        ...fails,
-    ];
+    return [header, summary, ...meta, ...fails];
 };
 exports.generateBlocks = generateBlocks;
 const generateFailures = async (summaryResults, maxNumberOfFailures) => {
     const maxNumberOfFailureLength = 650;
     const fails = [];
-    for (let i = 0; i < summaryResults.failures.length; i += 1) {
+    const numberOfFailuresToShow = Math.min(summaryResults.failures.length, maxNumberOfFailures);
+    for (let i = 0; i < numberOfFailuresToShow; i += 1) {
         const { failureReason, test } = summaryResults.failures[i];
         const formattedFailure = failureReason
             .substring(0, maxNumberOfFailureLength)
@@ -86,16 +54,15 @@ const generateFailures = async (summaryResults, maxNumberOfFailures) => {
         \n${formattedFailure}`,
             },
         });
-        if (i > maxNumberOfFailures) {
-            fails.push({
-                type: 'section',
-                text: {
-                    type: 'mrkdwn',
-                    text: `*There are too many failures to display - ${fails.length} out of ${summaryResults.failures.length} failures shown*`,
-                },
-            });
-            break;
-        }
+    }
+    if (summaryResults.failures.length > maxNumberOfFailures) {
+        fails.push({
+            type: 'section',
+            text: {
+                type: 'mrkdwn',
+                text: `*⚠️ There are too many failures to display - ${fails.length} out of ${summaryResults.failures.length} failures shown*`,
+            },
+        });
     }
     return [
         {

package/dist/src/ResultsParser.d.ts

@@ -1,5 +1,5 @@
 /// <reference types="node" />
-import { failure, SummaryResults } from '.';
+import { failure, flaky, pass, SummaryResults } from '.';
 export declare type testResult = {
     suiteName: string;
     name: string;
@@ -27,9 +27,14 @@ export declare type testSuite = {
 };
 export default class ResultsParser {
     private result;
-    constructor();
+    private separateFlakyTests;
+    constructor(options?: {
+        separateFlakyTests: boolean;
+    });
     getParsedResults(): Promise<SummaryResults>;
     getFailures(): Promise<Array<failure>>;
+    getFlakes(): Promise<Array<flaky>>;
+    getPasses(): Promise<Array<pass>>;
     static getTestName(failedTest: any): any;
     updateResults(data: {
         testSuite: any;
@@ -50,4 +55,7 @@ export default class ResultsParser {
         projectName: string;
         browser: string;
     };
+    /** removes tests from the passed array that only passed on a retry (flaky).
+     * Does not modify param passed, returns a new passed array. */
+    doSeparateFlakyTests(passes: Array<pass>, flakes: Array<flaky>): pass[];
 }

package/dist/src/ResultsParser.js

@@ -8,14 +8,22 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 class ResultsParser {
     result;
-    constructor() {
+    separateFlakyTests;
+    constructor(options = { separateFlakyTests: false }) {
         this.result = [];
+        this.separateFlakyTests = options.separateFlakyTests;
     }
     async getParsedResults() {
         const failures = await this.getFailures();
+        const flakes = await this.getFlakes();
+        let passes = await this.getPasses();
+        if (this.separateFlakyTests) {
+            passes = this.doSeparateFlakyTests(passes, flakes);
+        }
         const summary = {
-            passed: 0,
+            passed: passes.length,
             failed: failures.length,
+            flaky: this.separateFlakyTests ? flakes.length : undefined,
             skipped: 0,
             failures,
             tests: [],
@@ -23,10 +31,7 @@ class ResultsParser {
         for (const suite of this.result) {
             summary.tests = summary.tests.concat(suite.testSuite.tests);
             for (const test of suite.testSuite.tests) {
-                if (test.status === 'passed') {
-                    summary.passed += 1;
-                }
-                else if (test.status === 'skipped') {
+                if (test.status === 'skipped') {
                     summary.skipped += 1;
                 }
             }
@@ -50,6 +55,33 @@ class ResultsParser {
         }
         return failures;
     }
+    async getFlakes() {
+        const flaky = [];
+        for (const suite of this.result) {
+            for (const test of suite.testSuite.tests) {
+                if (test.status === 'passed' && test.retry > 0) {
+                    flaky.push({
+                        test: ResultsParser.getTestName(test),
+                        retry: test.retry,
+                    });
+                }
+            }
+        }
+        return flaky;
+    }
+    async getPasses() {
+        const passes = [];
+        for (const suite of this.result) {
+            for (const test of suite.testSuite.tests) {
+                if (test.status === 'passed') {
+                    passes.push({
+                        test: ResultsParser.getTestName(test),
+                    });
+                }
+            }
+        }
+        return passes;
+    }
     static getTestName(failedTest) {
         const testName = failedTest.name;
         if (failedTest.browser && failedTest.projectName) {
@@ -122,5 +154,17 @@ class ResultsParser {
             browser: '',
         };
     }
+    /** removes tests from the passed array that only passed on a retry (flaky).
+     * Does not modify param passed, returns a new passed array. */
+    doSeparateFlakyTests(passes, flakes) {
+        const _passes = new Map();
+        for (const pass of passes) {
+            _passes.set(pass.test, pass);
+        }
+        for (const flake of flakes) {
+            _passes.delete(flake.test);
+        }
+        return [..._passes.values()];
+    }
 }
 exports.default = ResultsParser;

package/dist/src/SlackReporter.d.ts

@@ -15,6 +15,7 @@ declare class SlackReporter implements Reporter {
     private proxy;
     private browsers;
     private suite;
+    private separateFlaky;
     logs: string[];
     onBegin(fullConfig: FullConfig, suite: Suite): void;
     onTestEnd(test: TestCase, result: TestResult): void;

package/dist/src/SlackReporter.js

@@ -22,6 +22,7 @@ class SlackReporter {
     proxy;
     browsers = [];
     suite;
+    separateFlaky = false;
     logs = [];
     onBegin(fullConfig, suite) {
         this.suite = suite;
@@ -53,8 +54,11 @@ class SlackReporter {
             this.showInThread = slackReporterConfig.showInThread || false;
             this.slackLogLevel = slackReporterConfig.slackLogLevel || web_api_1.LogLevel.DEBUG;
             this.proxy = slackReporterConfig.proxy || undefined;
+            this.separateFlaky = slackReporterConfig.separateFlaky || false;
         }
-        this.resultsParser = new ResultsParser_1.default();
+        this.resultsParser = new ResultsParser_1.default({
+            separateFlakyTests: this.separateFlaky,
+        });
     }
     // eslint-disable-next-line class-methods-use-this, no-unused-vars
     onTestEnd(test, result) {
@@ -108,12 +112,15 @@ class SlackReporter {
             // eslint-disable-next-line no-console
             console.log(JSON.stringify(result, null, 2));
             if (this.showInThread && resultSummary.failures.length > 0) {
-                await slackClient.attachDetailsToThread({
-                    channelIds: this.slackChannels,
-                    ts: result[0].ts,
-                    summaryResults: resultSummary,
-                    maxNumberOfFailures: this.maxNumberOfFailuresToShow,
-                });
+                for (let i = 0; i < result.length; i += 1) {
+                    // eslint-disable-next-line no-await-in-loop
+                    await slackClient.attachDetailsToThread({
+                        channelIds: [result[i].channel],
+                        ts: result[i].ts,
+                        summaryResults: resultSummary,
+                        maxNumberOfFailures: this.maxNumberOfFailuresToShow,
+                    });
+                }
             }
         }
     }

package/dist/src/custom_block/my_block.js

@@ -25,7 +25,7 @@ const generateCustomLayout = (summaryResults) => {
                 type: 'section',
                 text: {
                     type: 'mrkdwn',
-                    text: '*There are too many failures to display, view the full results in BuildKite*',
+                    text: '*⚠️ There are too many failures to display, view the full results in BuildKite*',
                 },
             });
             break;

package/dist/src/index.d.ts

@@ -2,6 +2,7 @@
 export declare type SummaryResults = {
     passed: number;
     failed: number;
+    flaky: number | undefined;
     skipped: number;
     failures: Array<failure>;
     meta?: Array<{
@@ -30,3 +31,10 @@ export declare type failure = {
     test: string;
     failureReason: string;
 };
+export declare type flaky = {
+    test: string;
+    retry: number;
+};
+export declare type pass = {
+    test: string;
+};

package/package.json

@@ -26,10 +26,11 @@
     "prettier": "prettier --write --loglevel warn \"**/**/*.ts\"",
     "pw": "nyc playwright test && nyc report --reporter=lcov",
     "build": "tsc -p ./tsconfig.json",
-    "lint": "npx eslint . --ext .ts"
+    "lint": "npx eslint . --ext .ts",
+    "lint-fix": "npx eslint . --ext .ts --fix"
   },
   "name": "playwright-slack-report",
-  "version": "1.1.22",
+  "version": "1.1.24",
   "main": "index.js",
   "types": "dist/index.d.ts",
   "repository": "git@github.com:ryanrosello-og/playwright-slack-report.git",