playwright-slack-report 1.1.23 → 1.1.24

package/README.md

@@ -179,6 +179,8 @@ A function that returns a layout object, this configuration is optional.  See se
 Same as **layout** above, but asynchronous in that it returns a promise.
 ### **maxNumberOfFailuresToShow**
 Limits the number of failures shown in the Slack message, defaults to 10.
+### **separateFlaky**
+Don't consider flaky tests as passed, instead output them separately. Flaky tests are tests that passed but not on the first try.  This defaults to `false` but it is highly recommended that you switch this on.
 ### **slackOAuthToken**
 Instead of providing an environment variable `SLACK_BOT_USER_OAUTH_TOKEN` you can specify the token in the config in the `slackOAuthToken` field.
 ### **slackLogLevel** (default LogLevel.DEBUG)

package/dist/src/LayoutGenerator.js

@@ -14,7 +14,9 @@ const generateBlocks = async (summaryResults, maxNumberOfFailures) => {
         type: 'section',
         text: {
             type: 'mrkdwn',
-            text: `✅ *${summaryResults.passed}* | ❌ *${summaryResults.failed}* | ⏩ *${summaryResults.skipped}*`,
+            text: `✅ *${summaryResults.passed}* | ❌ *${summaryResults.failed}* |${summaryResults.flaky !== undefined
+                ? ` 🟡 *${summaryResults.flaky}* | `
+                : ' '}⏩ *${summaryResults.skipped}*`,
         },
     };
     const fails = await generateFailures(summaryResults, maxNumberOfFailures);
@@ -30,12 +32,7 @@ const generateBlocks = async (summaryResults, maxNumberOfFailures) => {
             });
         }
     }
-    return [
-        header,
-        summary,
-        ...meta,
-        ...fails,
-    ];
+    return [header, summary, ...meta, ...fails];
 };
 exports.generateBlocks = generateBlocks;
 const generateFailures = async (summaryResults, maxNumberOfFailures) => {

package/dist/src/ResultsParser.d.ts

@@ -1,5 +1,5 @@
 /// <reference types="node" />
-import { failure, SummaryResults } from '.';
+import { failure, flaky, pass, SummaryResults } from '.';
 export declare type testResult = {
     suiteName: string;
     name: string;
@@ -27,9 +27,14 @@ export declare type testSuite = {
 };
 export default class ResultsParser {
     private result;
-    constructor();
+    private separateFlakyTests;
+    constructor(options?: {
+        separateFlakyTests: boolean;
+    });
     getParsedResults(): Promise<SummaryResults>;
     getFailures(): Promise<Array<failure>>;
+    getFlakes(): Promise<Array<flaky>>;
+    getPasses(): Promise<Array<pass>>;
     static getTestName(failedTest: any): any;
     updateResults(data: {
         testSuite: any;
@@ -50,4 +55,7 @@ export default class ResultsParser {
         projectName: string;
         browser: string;
     };
+    /** removes tests from the passed array that only passed on a retry (flaky).
+     * Does not modify param passed, returns a new passed array. */
+    doSeparateFlakyTests(passes: Array<pass>, flakes: Array<flaky>): pass[];
 }

package/dist/src/ResultsParser.js

@@ -8,14 +8,22 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 class ResultsParser {
     result;
-    constructor() {
+    separateFlakyTests;
+    constructor(options = { separateFlakyTests: false }) {
         this.result = [];
+        this.separateFlakyTests = options.separateFlakyTests;
     }
     async getParsedResults() {
         const failures = await this.getFailures();
+        const flakes = await this.getFlakes();
+        let passes = await this.getPasses();
+        if (this.separateFlakyTests) {
+            passes = this.doSeparateFlakyTests(passes, flakes);
+        }
         const summary = {
-            passed: 0,
+            passed: passes.length,
             failed: failures.length,
+            flaky: this.separateFlakyTests ? flakes.length : undefined,
             skipped: 0,
             failures,
             tests: [],
@@ -23,10 +31,7 @@ class ResultsParser {
         for (const suite of this.result) {
             summary.tests = summary.tests.concat(suite.testSuite.tests);
             for (const test of suite.testSuite.tests) {
-                if (test.status === 'passed') {
-                    summary.passed += 1;
-                }
-                else if (test.status === 'skipped') {
+                if (test.status === 'skipped') {
                     summary.skipped += 1;
                 }
             }
@@ -50,6 +55,33 @@ class ResultsParser {
         }
         return failures;
     }
+    async getFlakes() {
+        const flaky = [];
+        for (const suite of this.result) {
+            for (const test of suite.testSuite.tests) {
+                if (test.status === 'passed' && test.retry > 0) {
+                    flaky.push({
+                        test: ResultsParser.getTestName(test),
+                        retry: test.retry,
+                    });
+                }
+            }
+        }
+        return flaky;
+    }
+    async getPasses() {
+        const passes = [];
+        for (const suite of this.result) {
+            for (const test of suite.testSuite.tests) {
+                if (test.status === 'passed') {
+                    passes.push({
+                        test: ResultsParser.getTestName(test),
+                    });
+                }
+            }
+        }
+        return passes;
+    }
     static getTestName(failedTest) {
         const testName = failedTest.name;
         if (failedTest.browser && failedTest.projectName) {
@@ -122,5 +154,17 @@ class ResultsParser {
             browser: '',
         };
     }
+    /** removes tests from the passed array that only passed on a retry (flaky).
+     * Does not modify param passed, returns a new passed array. */
+    doSeparateFlakyTests(passes, flakes) {
+        const _passes = new Map();
+        for (const pass of passes) {
+            _passes.set(pass.test, pass);
+        }
+        for (const flake of flakes) {
+            _passes.delete(flake.test);
+        }
+        return [..._passes.values()];
+    }
 }
 exports.default = ResultsParser;

package/dist/src/SlackReporter.d.ts

@@ -15,6 +15,7 @@ declare class SlackReporter implements Reporter {
     private proxy;
     private browsers;
     private suite;
+    private separateFlaky;
     logs: string[];
     onBegin(fullConfig: FullConfig, suite: Suite): void;
     onTestEnd(test: TestCase, result: TestResult): void;

package/dist/src/SlackReporter.js

@@ -22,6 +22,7 @@ class SlackReporter {
     proxy;
     browsers = [];
     suite;
+    separateFlaky = false;
     logs = [];
     onBegin(fullConfig, suite) {
         this.suite = suite;
@@ -53,8 +54,11 @@ class SlackReporter {
             this.showInThread = slackReporterConfig.showInThread || false;
             this.slackLogLevel = slackReporterConfig.slackLogLevel || web_api_1.LogLevel.DEBUG;
             this.proxy = slackReporterConfig.proxy || undefined;
+            this.separateFlaky = slackReporterConfig.separateFlaky || false;
         }
-        this.resultsParser = new ResultsParser_1.default();
+        this.resultsParser = new ResultsParser_1.default({
+            separateFlakyTests: this.separateFlaky,
+        });
     }
     // eslint-disable-next-line class-methods-use-this, no-unused-vars
     onTestEnd(test, result) {

package/dist/src/index.d.ts

@@ -2,6 +2,7 @@
 export declare type SummaryResults = {
     passed: number;
     failed: number;
+    flaky: number | undefined;
     skipped: number;
     failures: Array<failure>;
     meta?: Array<{
@@ -30,3 +31,10 @@ export declare type failure = {
     test: string;
     failureReason: string;
 };
+export declare type flaky = {
+    test: string;
+    retry: number;
+};
+export declare type pass = {
+    test: string;
+};

package/package.json

@@ -26,10 +26,11 @@
     "prettier": "prettier --write --loglevel warn \"**/**/*.ts\"",
     "pw": "nyc playwright test && nyc report --reporter=lcov",
     "build": "tsc -p ./tsconfig.json",
-    "lint": "npx eslint . --ext .ts"
+    "lint": "npx eslint . --ext .ts",
+    "lint-fix": "npx eslint . --ext .ts --fix"
   },
   "name": "playwright-slack-report",
-  "version": "1.1.23",
+  "version": "1.1.24",
   "main": "index.js",
   "types": "dist/index.d.ts",
   "repository": "git@github.com:ryanrosello-og/playwright-slack-report.git",