squally-reporter 0.3.0 → 0.5.0

package/README.md

@@ -1,121 +1,182 @@
-# squally-reporter
-
-Playwright reporter that sends test results to your Squally dashboard.
-
-Learn more at [squally-app.vercel.app](https://squally-app.vercel.app).
-
-## Requirements
-
-- Node.js >= 22.19.0 (required by the `undici` dependency)
-- `@playwright/test` >= 1.30.0
-
-## Install
-
-```bash
-npm install squally-reporter
-```
-
-## Config
-
-Set two env vars (e.g. in CI secrets or `.env`):
-
-```bash
-SQUALLY_API_URL=https://squally-app.vercel.app
-SQUALLY_API_KEY=your-project-api-key
-```
-
-Find your API key on your project's Settings page in the Squally dashboard.
-
-Then add the reporter to `playwright.config.ts`:
-
-```ts
-import { defineConfig } from "@playwright/test";
-
-export default defineConfig({
-  reporter: [
-    ["list"],
-    ["squally-reporter"],
-  ],
-});
-```
-
-Alternatively, pass config explicitly instead of using env vars:
-
-```ts
-reporter: [
-  ["squally-reporter", { apiUrl: "https://squally-app.vercel.app", apiKey: "..." }],
-],
-```
-
-That's it - run your tests as usual (`npx playwright test`) and results will
-be sent to Squally after the run finishes.
-
-## Screenshot/trace upload
-
-If your `playwright.config.ts` captures screenshots or traces, the reporter
-uploads them to Squally so they show up next to the failing test in the
-dashboard - no separate setup beyond the two env vars above.
-
-**Only from attempts worth looking at:** screenshots/traces are uploaded
-only for attempts that **failed**, or that **needed a retry** (even if that
-retry then passed) - never for an attempt that passed on its first try,
-regardless of how `screenshot`/`trace` are configured. Nobody looks at a
-screenshot of a test that just passed, and uploading one anyway would only
-eat into your organization's storage for no benefit.
-
-**You need to actually enable capture in `playwright.config.ts`** - without
-it, Playwright doesn't produce any attachments, so there's nothing for the
-reporter to find or upload (no error either way, it's a silent no-op):
-
-```ts
-export default defineConfig({
-  use: {
-    screenshot: "only-on-failure",
-    trace: "retain-on-failure",
-  },
-  // ...
-});
-```
-
-**Limits:**
-- 50 MB per individual file. Larger files are skipped (logged as a
-  `console.warn`, the test result itself is still reported as usual).
-- A total storage cap per organization, depending on your Squally plan.
-  Once reached, further uploads are skipped the same way until you either
-  free up space or upgrade.
-
-Upload is best-effort and fully non-blocking, same principle as the rest of
-this reporter - a failed or skipped upload never fails your test run or
-changes its exit code, it only logs a `console.warn` explaining what
-happened.
-
-## Tested revision (optional)
-
-If your test-automation code lives in the same repo as the application it
-tests, Squally can already tell which commit was tested from the CI
-provider's own commit env vars. If they're **separate repos** (e.g. a
-dedicated E2E-test repo pointed at a deployed app), set one of the following
-so Squally knows which application state was actually tested:
-
-```bash
-# Same repo as the application under test - Squally will trust this CI
-# job's own commit (GITHUB_SHA / CI_COMMIT_SHA) as the tested revision.
-SAME_REPO_AS_APP=true
-
-# Separate repos - pass the tested revision explicitly (a commit SHA, tag,
-# or branch name/ref).
-TESTED_REVISION=v1.2.0
-```
-
-Neither variable is required - without them, test results are still sent
-and stored exactly as before.
-
-## Support
-
-Found a bug or have a question? Please open an issue on
-[GitHub Issues](https://github.com/squallydev-admin/squally-reporter/issues) -
-that's the primary way to report problems or ask questions.
-
-## License
-
-MIT - see [LICENSE](./LICENSE).
+# squally-reporter
+
+Playwright reporter that sends test results to your Squally dashboard.
+
+Learn more at [squally-app.vercel.app](https://squally-app.vercel.app).
+
+## Requirements
+
+- Node.js >= 22.19.0 (required by the `undici` dependency)
+- `@playwright/test` >= 1.30.0
+
+## Install
+
+```bash
+npm install -D squally-reporter
+```
+
+## Config
+
+Set two env vars (e.g. in CI secrets or `.env`):
+
+```bash
+SQUALLY_API_URL=https://squally-app.vercel.app
+SQUALLY_API_KEY=your-project-api-key
+```
+
+Find your API key on your project's Settings page in the Squally dashboard.
+
+Then add the reporter to `playwright.config.ts`:
+
+```ts
+import { defineConfig } from "@playwright/test";
+
+export default defineConfig({
+  reporter: [
+    ["list"],
+    ["squally-reporter"],
+  ],
+});
+```
+
+Alternatively, pass config explicitly instead of using env vars - **only if
+`apiKey` itself comes from an env var or secret store at runtime, never as a
+literal string in a committed file:**
+
+```ts
+reporter: [
+  ["squally-reporter", { apiUrl: "https://squally-app.vercel.app", apiKey: process.env.SQUALLY_API_KEY }],
+],
+```
+
+That's it - run your tests as usual (`npx playwright test`) and results will
+be sent to Squally after the run finishes.
+
+## Screenshot/trace upload
+
+If your `playwright.config.ts` captures screenshots or traces, the reporter
+uploads them to Squally so they show up next to the failing test in the
+dashboard - no separate setup beyond the two env vars above.
+
+**Only from attempts worth looking at:** screenshots/traces are uploaded
+only for attempts that **failed**, or that **needed a retry** (even if that
+retry then passed) - never for an attempt that passed on its first try,
+regardless of how `screenshot`/`trace` are configured. Nobody looks at a
+screenshot of a test that just passed, and uploading one anyway would only
+eat into your organization's storage for no benefit.
+
+**You need to actually enable capture in `playwright.config.ts`** - without
+it, Playwright doesn't produce any attachments, so there's nothing for the
+reporter to find or upload (no error either way, it's a silent no-op):
+
+```ts
+export default defineConfig({
+  use: {
+    screenshot: "only-on-failure",
+    trace: "retain-on-failure",
+  },
+  // ...
+});
+```
+
+**Limits:**
+- 50 MB per individual file. Larger files are skipped (logged as a
+  `console.warn`, the test result itself is still reported as usual).
+- A total storage cap per organization, depending on your Squally plan.
+  Once reached, further uploads are skipped the same way until you either
+  free up space or upgrade.
+- Two minutes per upload, and 30 seconds for the calls that surround it. An
+  unreachable or very slow server means a skipped upload and a `console.warn`,
+  not a CI job waiting indefinitely.
+
+Upload is best-effort and fully non-blocking, same principle as the rest of
+this reporter - a failed or skipped upload never fails your test run or
+changes its exit code, it only logs a `console.warn` explaining what
+happened.
+
+## How results reach Squally
+
+Results stream to Squally as your suite runs, not all at once at the end:
+each test's result is sent in a small batch (at most every 25 results or 2
+seconds, whichever comes first), so the "running" view in the dashboard stays
+close to real time even for a long suite. There is no longer a single-request
+size limit that could discard a whole run - a very large suite simply takes
+more, smaller requests. As a courtesy the reporter still warns if a run's
+result count climbs well past 5,000, since that usually means retries are
+configured higher than intended rather than a suite that's actually that
+large.
+
+If Squally is unreachable partway through, results already sent stay in the
+dashboard; only what could not be delivered is lost, and your test run's exit
+code is never affected either way.
+
+## Sharding
+
+**Handled automatically - no extra setup needed.** Each shard is still a
+separate process with its own reporter instance, but Squally now correlates
+them itself: all shards of one CI run are grouped together and shown as one
+run once every shard has finished, with the flip side of that guarantee being
+that the run only completes once every shard *has* reported.
+
+Correlation is based on an id read from your CI provider's own environment,
+so it costs you nothing on GitHub Actions, GitLab CI or CircleCI. If your
+provider is not one of those (or you shard locally across several separate
+`--shard` invocations), set `SQUALLY_CI_RUN_ID` to the same value for every
+shard of one run - anything unique per CI run and shared by its shards works,
+e.g. a build number your CI system already exposes.
+
+**`playwright merge-reports` is not part of this and is not recommended** for
+getting a single run in Squally - the automatic correlation above is the
+supported path, and produces correct per-shard timing/branch/commit
+information that a merge job cannot (see the historical note below). Nothing
+stops you from using `merge-reports` for Playwright's own HTML report if you
+want that separately; just don't point this reporter at the merged result
+expecting Squally to treat it specially.
+
+<details>
+<summary>Why not just use `merge-reports`, historically</summary>
+
+An earlier version of this reporter recommended `playwright merge-reports` as
+the way to get one Squally run out of a sharded job. That path has a real
+problem: branch, commit, commit message and author are all read on whichever
+machine the reporter actually runs on - during a merge that is the merge job,
+not the shards - and without a checkout there (and without the CI provider's
+own commit variables) they all come out empty. A run with no branch is never
+counted as a run on your stable branch, which is what Squally's broken/healthy
+verdicts are based on - so a "merged" run could exist in the dashboard while
+silently contributing nothing to flakiness detection. The automatic
+correlation above reads that information from each shard's own environment
+instead, so this problem does not arise.
+
+</details>
+
+## Tested revision (optional)
+
+If your test-automation code lives in the same repo as the application it
+tests, Squally can already tell which commit was tested from the CI
+provider's own commit env vars. If they're **separate repos** (e.g. a
+dedicated E2E-test repo pointed at a deployed app), set one of the following
+so Squally knows which application state was actually tested:
+
+```bash
+# Same repo as the application under test - Squally will trust this CI
+# job's own commit (GITHUB_SHA / CI_COMMIT_SHA) as the tested revision.
+SAME_REPO_AS_APP=true
+
+# Separate repos - pass the tested revision explicitly (a commit SHA, tag,
+# or branch name/ref).
+TESTED_REVISION=v1.2.0
+```
+
+Neither variable is required - without them, test results are still sent
+and stored exactly as before.
+
+## Support
+
+Found a bug or have a question? Please open an issue on
+[GitHub Issues](https://github.com/squallydev-admin/squally-reporter/issues) -
+that's the primary way to report problems or ask questions.
+
+## License
+
+MIT - see [LICENSE](./LICENSE).

package/dist/index.d.ts

@@ -1,20 +1,111 @@
-import type { FullConfig, FullResult, Reporter, Suite, TestCase, TestResult } from "@playwright/test/reporter";
+import type { FullConfig, FullResult, Reporter, Suite, TestCase, TestError, TestResult } from "@playwright/test/reporter";
 interface SquallyReporterOptions {
     apiUrl?: string;
     apiKey?: string;
 }
+interface AppendErrorEntry {
+    message: string;
+    stack?: string;
+    snippet?: string;
+    location?: {
+        file: string;
+        line: number;
+        column: number;
+    };
+}
+interface AppendResultEntry {
+    testName: string;
+    filePath: string | null;
+    status: "passed" | "failed" | "skipped";
+    retryCount: number;
+    repeatIndex: number;
+    durationMs: number | null;
+    errors?: AppendErrorEntry[];
+    consoleOutput: string | null;
+    tags?: string[];
+    browserName: string | null;
+    errorContext: string | null;
+}
+type ArtifactType = "screenshot" | "trace";
+declare function truncate(value: string, max: number): string;
+declare function truncateOrNull(value: string | null, max: number): string | null;
+declare function mapStatus(status: TestResult["status"]): AppendResultEntry["status"];
+declare function detectCiRunId(env: NodeJS.ProcessEnv, generateFallback: () => string): string;
+interface ProjectSuiteLike {
+    project(): {
+        name: string;
+    } | undefined;
+}
+declare function resolveProjectName(projectSuites: readonly ProjectSuiteLike[]): string | null;
+declare function normalizeRevision(raw: string): string;
+interface ResolvedRevision {
+    revision: string;
+    source: "ci_native" | "passed_param";
+}
+declare function resolveTestedRevision(): ResolvedRevision | null;
+declare function toPosixPath(p: string): string;
+declare function stripAnsi(value: string): string;
+declare function maskSecrets(value: string): string;
+declare function prepareConsoleOutput(value: string | null): string | null;
+declare function mapErrors(errors: readonly TestError[]): AppendErrorEntry[] | undefined;
+declare function attemptQualifiesForArtifacts(mappedStatus: AppendResultEntry["status"], retryCount: number): boolean;
+declare function artifactKey(testName: string, filePath: string, retryCount: number, type: ArtifactType): string;
+declare function parseRetryAfterMs(headerValue: string | null, nowMs: number): number | null;
+declare function computeBackoffMs(retryNumber: number): number;
 export default class SquallyReporter implements Reporter {
     private apiUrl;
     private apiKey;
     private enabled;
     private startedAt;
-    private results;
+    private dispatcher;
+    private ciRunId;
+    private shardIndex;
+    private shardTotal;
+    private projectName;
+    private runTags;
+    private runConfig;
+    private buffer;
     private artifacts;
+    private sendingDisabled;
+    private flushTimerHandle;
+    private inFlightFlushes;
+    private runIdPromise;
+    private runId;
     constructor(options?: SquallyReporterOptions);
-    onBegin(_config: FullConfig, _suite: Suite): void;
+    onBegin(config: FullConfig, suite: Suite): void;
+    private resolveRunConfig;
     onTestEnd(test: TestCase, result: TestResult): void;
-    onEnd(result: FullResult): Promise<void>;
+    onEnd: (result: FullResult) => Promise<void>;
+    private drainAndComplete;
+    private armFlushTimer;
+    private stopFlushTimer;
+    private giveUp;
+    private triggerFlush;
+    private flush;
+    private sendBatch;
+    private beginRun;
+    private completeRun;
+    private fetchRules;
+    private requestJson;
     private uploadArtifacts;
     private presignAndUpload;
 }
+export declare const __internal: {
+    truncate: typeof truncate;
+    truncateOrNull: typeof truncateOrNull;
+    mapStatus: typeof mapStatus;
+    stripAnsi: typeof stripAnsi;
+    normalizeRevision: typeof normalizeRevision;
+    resolveTestedRevision: typeof resolveTestedRevision;
+    toPosixPath: typeof toPosixPath;
+    detectCiRunId: typeof detectCiRunId;
+    resolveProjectName: typeof resolveProjectName;
+    maskSecrets: typeof maskSecrets;
+    prepareConsoleOutput: typeof prepareConsoleOutput;
+    mapErrors: typeof mapErrors;
+    parseRetryAfterMs: typeof parseRetryAfterMs;
+    computeBackoffMs: typeof computeBackoffMs;
+    attemptQualifiesForArtifacts: typeof attemptQualifiesForArtifacts;
+    artifactKey: typeof artifactKey;
+};
 export {};