proxymail-qa 0.7.0 → 0.8.1

package/README.md

@@ -52,6 +52,24 @@ test('sign-up sends a working code', async ({ page, inbox }) => {
 });
 ```
 
+In a run (the GitHub Action, or `startRun` in `globalSetup`), the report can say
+which test each email came from. Use the `qa` fixture from
+`proxymail-qa/playwright`, a client that names its test in every call, and add
+the reporter, which records failed and flaky tests and the totals:
+
+```ts
+// playwright.config.ts
+reporter: [['list'], ['proxymail-qa/playwright/reporter']],
+
+// signup.spec.ts
+import { test, expect } from 'proxymail-qa/playwright';
+
+test('sign-up sends one code', async ({ page, qa }) => {
+  const { address } = await qa.createAddress();
+  // …
+});
+```
+
 ## Cypress
 
 Register tasks in `cypress.config.ts`, since Cypress runs them in Node:

package/dist/index.d.ts

@@ -3,6 +3,19 @@ export interface ClientOptions {
     baseUrl?: string;
     fetch?: typeof fetch;
     run?: string | null;
+    test?: string | null;
+}
+export interface TestResultInput {
+    name: string;
+    status: 'passed' | 'failed' | 'flaky' | 'skipped';
+    durationMs?: number;
+    error?: string;
+}
+export interface TestTotals {
+    passed: number;
+    failed: number;
+    skipped?: number;
+    flaky?: number;
 }
 export interface Persona {
     first_name: string | null;
@@ -190,7 +203,7 @@ export interface CountOptions {
     subtype?: CountSubtype | CountSubtype[];
 }
 export type CountSubtype = 'otp' | 'coupon' | 'verify' | 'magic_signin' | 'invite' | 'reset' | 'other' | 'any';
-export type CheckId = 'spf' | 'dkim' | 'dmarc' | 'local_links' | 'plain_http' | 'expected_host' | 'expiry' | 'plain_text' | 'size' | 'images_only' | 'expected_text' | 'language';
+export type CheckId = 'spf' | 'dkim' | 'dmarc' | 'local_links' | 'plain_http' | 'expected_host' | 'expiry' | 'plain_text' | 'size' | 'images_only' | 'layout' | 'expected_text' | 'language';
 export interface EmailCheck {
     check: CheckId;
     status: 'pass' | 'fail' | 'warn' | 'skip';
@@ -266,7 +279,9 @@ export declare class ProxyMailQA {
     private readonly baseUrl;
     private readonly fetcher;
     run: string | null;
+    readonly test: string | null;
     constructor(options?: ClientOptions);
+    forTest(name: string): ProxyMailQA;
     private request;
     project(): Promise<Record<string, unknown>>;
     createAddress(input: CreateAddressInput & {
@@ -329,6 +344,8 @@ export declare class ProxyMailQA {
     getRun(id?: string): Promise<Run>;
     runReport(id?: string): Promise<string>;
     listRuns(limit?: number): Promise<Run[]>;
+    recordTest(result: TestResultInput, run?: string): Promise<void>;
+    recordTestTotals(totals: TestTotals, run?: string): Promise<void>;
     addRunNote(text: string, options?: {
         failed?: boolean;
         address?: string;

package/dist/index.js

@@ -42,15 +42,25 @@ export class ProxyMailQA {
     fetcher;
     // The run this client's calls add to, if any.
     run;
+    // The test they belong to, if any.
+    test;
     constructor(options = {}) {
         const env = globalThis.process?.env;
         const apiKey = options.apiKey ?? env?.PROXYMAIL_QA_API_KEY;
         if (!apiKey)
             throw new Error('Pass apiKey or set PROXYMAIL_QA_API_KEY. Create a key in Project settings → API keys.');
         this.apiKey = apiKey;
-        this.baseUrl = (options.baseUrl ?? 'https://api.proxymail.qa').replace(/\/+$/, '');
+        // PROXYMAIL_QA_API_URL, as the Python helper reads it, may end in /v1.
+        this.baseUrl = (options.baseUrl ?? (env?.PROXYMAIL_QA_API_URL?.trim().replace(/\/v1\/?$/, '') || 'https://api.proxymail.qa')).replace(/\/+$/, '');
         this.fetcher = options.fetch ?? globalThis.fetch.bind(globalThis);
         this.run = options.run === undefined ? env?.PROXYMAIL_QA_RUN?.trim() || null : options.run;
+        this.test = options.test?.trim() || null;
+    }
+    // A client for one test: the same key and run, and every call says which
+    // test it belongs to, so the run's report shows which test each email came
+    // from. proxymail-qa/playwright's qa fixture does this for you.
+    forTest(name) {
+        return new ProxyMailQA({ apiKey: this.apiKey, baseUrl: this.baseUrl, fetch: this.fetcher, run: this.run, test: name });
     }
     // retry: this call is asked again if nothing arrives, so the run only
     // records the last try's "nothing arrived".
@@ -61,6 +71,8 @@ export class ProxyMailQA {
                 Authorization: `Bearer ${this.apiKey}`,
                 ...(body === undefined ? {} : { 'Content-Type': 'application/json' }),
                 ...(this.run ? { 'Proxy-Mail-QA-Run': this.run, ...(retry ? { 'Proxy-Mail-QA-Run-Retry': 'true' } : {}) } : {}),
+                // Percent-encoded: a test's name can hold characters headers can't.
+                ...(this.run && this.test ? { 'Proxy-Mail-QA-Test': encodeURIComponent(this.test) } : {}),
             },
             body: body === undefined ? undefined : JSON.stringify(body),
         });
@@ -421,6 +433,23 @@ export class ProxyMailQA {
     }
     // A line of your own in the run, such as a test that failed for a reason
     // of its own. failed: true fails the run.
+    // A test's result in the run (a failed one fails it), from a test runner's
+    // reporter: proxymail-qa/playwright/reporter sends failed and flaky tests,
+    // then the totals.
+    recordTest(result, run) {
+        const id = run ?? this.requireRun();
+        return this.request('POST', `/runs/${encodeURIComponent(id)}/tests`, {
+            name: result.name,
+            status: result.status,
+            duration_ms: result.durationMs,
+            error: result.error,
+        });
+    }
+    // How the suite went, for the top of the run's report.
+    recordTestTotals(totals, run) {
+        const id = run ?? this.requireRun();
+        return this.request('POST', `/runs/${encodeURIComponent(id)}/tests`, { totals });
+    }
     addRunNote(text, options = {}) {
         const id = options.run ?? this.requireRun();
         return this.request('POST', `/runs/${encodeURIComponent(id)}/notes`, { text, failed: options.failed ?? false, address: options.address });

package/dist/playwright-reporter.d.ts

@@ -0,0 +1,11 @@
+import type { FullResult, Reporter, Suite } from '@playwright/test/reporter';
+import { type ClientOptions } from './index.js';
+export default class ProxyMailQAReporter implements Reporter {
+    private readonly options;
+    private suite;
+    constructor(options?: Pick<ClientOptions, 'apiKey' | 'baseUrl'>);
+    onBegin(_config: unknown, suite: Suite): void;
+    onEnd(_result: FullResult): Promise<void>;
+    printsToStdio(): boolean;
+    private client;
+}

package/dist/playwright-reporter.js

@@ -0,0 +1,82 @@
+import { ProxyMailQA } from './index.js';
+import { testName } from './test-name.js';
+// Enough for a report; the rest are in Playwright's own.
+const MAX_RESULTS = 200;
+export default class ProxyMailQAReporter {
+    options;
+    suite = null;
+    constructor(options = {}) {
+        this.options = options;
+    }
+    onBegin(_config, suite) {
+        this.suite = suite;
+    }
+    async onEnd(_result) {
+        const qa = this.client();
+        if (!qa || !this.suite)
+            return;
+        const totals = { passed: 0, failed: 0, skipped: 0, flaky: 0 };
+        const results = [];
+        for (const test of this.suite.allTests()) {
+            const outcome = test.outcome();
+            if (outcome === 'expected')
+                totals.passed++;
+            else if (outcome === 'skipped')
+                totals.skipped++;
+            else {
+                const flaky = outcome === 'flaky';
+                if (flaky)
+                    totals.flaky++;
+                else
+                    totals.failed++;
+                const last = test.results.at(-1);
+                results.push({
+                    name: nameOf(test),
+                    status: flaky ? 'flaky' : 'failed',
+                    durationMs: last?.duration,
+                    error: flaky ? undefined : withStep(failedStep(last?.steps ?? []), last?.error?.message),
+                });
+            }
+        }
+        // Failures never fail the test run itself: the report is evidence.
+        for (const result of results.slice(0, MAX_RESULTS))
+            await qa.recordTest(result).catch(() => { });
+        await qa.recordTestTotals(totals).catch(() => { });
+    }
+    printsToStdio() {
+        return false;
+    }
+    client() {
+        try {
+            const qa = new ProxyMailQA(this.options);
+            return qa.run ? qa : null;
+        }
+        catch {
+            // No API key: nothing to report to.
+            return null;
+        }
+    }
+}
+// titlePath: the root, the project, the file, then describe blocks and the title.
+function nameOf(test) {
+    const [, project = '', , ...path] = test.titlePath();
+    return testName(project, path);
+}
+// The innermost step that failed: a test.step of the test's own, or the
+// browser action or assertion inside it, such as a click or an expect.
+function failedStep(steps) {
+    for (const step of steps) {
+        if (!step.error)
+            continue;
+        return failedStep(step.steps) ?? step;
+    }
+    return null;
+}
+// "At “Fill in the code” › locator.fill(…): Error: …", so the report says
+// which browser step failed as well as why.
+function withStep(step, message) {
+    if (!step)
+        return message;
+    const path = step.titlePath().filter(Boolean).slice(-2).join(' › ');
+    return `At “${path}”: ${message ?? 'failed'}`;
+}

package/dist/playwright.d.ts

@@ -0,0 +1,5 @@
+import { ProxyMailQA } from './index.js';
+export declare const test: import("playwright/test").TestType<import("playwright/test").PlaywrightTestArgs & import("playwright/test").PlaywrightTestOptions & {
+    qa: ProxyMailQA;
+}, import("playwright/test").PlaywrightWorkerArgs & import("playwright/test").PlaywrightWorkerOptions>;
+export { expect } from '@playwright/test';

package/dist/playwright.js

@@ -0,0 +1,25 @@
+// Playwright tests with Proxy Mail QA. The qa fixture is a client for the
+// test it's in: in a run (PROXYMAIL_QA_RUN, set by the GitHub Action or by
+// startRun in globalSetup), each call says which test it belongs to, so the
+// run's report shows which test each email came from.
+//
+//   import { test, expect } from 'proxymail-qa/playwright';
+//
+//   test('sign-up sends one code', async ({ page, qa }) => {
+//     const { address } = await qa.createAddress();
+//     …
+//   });
+//
+// Add proxymail-qa/playwright/reporter to the config's reporters to record
+// failed tests and the totals in the run too.
+import { test as base } from '@playwright/test';
+import { ProxyMailQA } from './index.js';
+import { testName } from './test-name.js';
+export const test = base.extend({
+    // eslint-disable-next-line no-empty-pattern
+    qa: async ({}, use, testInfo) => {
+        // titlePath starts with the file.
+        await use(new ProxyMailQA().forTest(testName(testInfo.project.name, testInfo.titlePath.slice(1))));
+    },
+});
+export { expect } from '@playwright/test';

package/dist/test-name.d.ts

@@ -0,0 +1 @@
+export declare const testName: (project: string, path: readonly string[]) => string;

package/dist/test-name.js

@@ -0,0 +1,3 @@
+// A test's name as the run shows it, the same in the qa fixture and the
+// reporter: the project (when named), the describe blocks and the title.
+export const testName = (project, path) => [project, ...path].filter(Boolean).join(' › ');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "proxymail-qa",
-  "version": "0.7.0",
+  "version": "0.8.1",
   "description": "Disposable test inboxes for Playwright, Cypress and scripts: create an address, then wait for its verification code or link.",
   "license": "MIT",
   "author": "Proxy Mail QA <hello@proxymail.qa> (https://proxymail.qa)",
@@ -29,6 +29,14 @@
     ".": {
       "types": "./dist/index.d.ts",
       "default": "./dist/index.js"
+    },
+    "./playwright": {
+      "types": "./dist/playwright.d.ts",
+      "default": "./dist/playwright.js"
+    },
+    "./playwright/reporter": {
+      "types": "./dist/playwright-reporter.d.ts",
+      "default": "./dist/playwright-reporter.js"
     }
   },
   "files": [
@@ -46,7 +54,16 @@
     "typecheck": "tsc -p tsconfig.json --noEmit",
     "prepublishOnly": "npm run build && npm test"
   },
+  "peerDependencies": {
+    "@playwright/test": ">=1.40.0"
+  },
+  "peerDependenciesMeta": {
+    "@playwright/test": {
+      "optional": true
+    }
+  },
   "devDependencies": {
+    "@playwright/test": "^1.64.1",
     "typescript": "^5.6.0",
     "vitest": "^5.0.3"
   }