squally-reporter 0.4.0 → 0.5.0

package/README.md

@@ -39,11 +39,13 @@ export default defineConfig({
 });
 ```
 
-Alternatively, pass config explicitly instead of using env vars:
+Alternatively, pass config explicitly instead of using env vars - **only if
+`apiKey` itself comes from an env var or secret store at runtime, never as a
+literal string in a committed file:**
 
 ```ts
 reporter: [
-  ["squally-reporter", { apiUrl: "https://squally-app.vercel.app", apiKey: "..." }],
+  ["squally-reporter", { apiUrl: "https://squally-app.vercel.app", apiKey: process.env.SQUALLY_API_KEY }],
 ],
 ```
 
@@ -92,92 +94,61 @@ this reporter - a failed or skipped upload never fails your test run or
 changes its exit code, it only logs a `console.warn` explaining what
 happened.
 
-## Limits per run
+## How results reach Squally
 
-Squally accepts at most **5,000 test results in a single run**. Beyond that the
-whole report is rejected, so the reporter checks before sending and skips the
-upload with a warning rather than letting the server discard it after the fact.
-Nothing else about your test run is affected - this only means that one run
-does not appear in the dashboard.
+Results stream to Squally as your suite runs, not all at once at the end:
+each test's result is sent in a small batch (at most every 25 results or 2
+seconds, whichever comes first), so the "running" view in the dashboard stays
+close to real time even for a long suite. There is no longer a single-request
+size limit that could discard a whole run - a very large suite simply takes
+more, smaller requests. As a courtesy the reporter still warns if a run's
+result count climbs well past 5,000, since that usually means retries are
+configured higher than intended rather than a suite that's actually that
+large.
 
-**Retries count separately.** Each attempt is its own result, so a suite of
-2,000 tests having a bad day reaches the ceiling sooner than the test count
-suggests.
-
-If you run into this, the practical way around it today is to let separate
-parts of the suite report separately - for example as separate Playwright
-projects or separate CI jobs, each producing its own run. Splitting one report
-across several requests is not supported yet.
-
-There is also a hard **~4.5 MB limit on the request itself**, coming from the
-hosting platform rather than from Squally. Very large error messages, stack
-traces or console output can hit it before the 5,000 mark; the reporter already
-truncates each field to what the server accepts, which keeps normal runs well
-clear of it.
+If Squally is unreachable partway through, results already sent stay in the
+dashboard; only what could not be delivered is lost, and your test run's exit
+code is never affected either way.
 
 ## Sharding
 
-**Playwright's `--shard` is not merged automatically.** Each shard is a
-separate process with its own reporter instance, so each one reports its own
-run: a job split four ways shows up in Squally as four runs from the same
-commit, each holding only its own slice of the suite. The reporter says so in
-the log when it sees `--shard`.
-
-What that does and does not affect:
-
-- **Per-test flakiness stays correct.** A test lives in exactly one shard, so
-  it still contributes one data point per CI run, and its history, flip rate
-  and retry rate are unaffected.
-- **Run-level numbers are off.** The run count, "runs per day", and each run's
-  own pass/fail (it only covers its slice) all reflect shards rather than CI
-  runs.
-
-### Reporting sharded runs as one run
-
-Use Playwright's own blob reports: have each shard write a blob, then merge
-them in one final job and let this reporter run on the merged result. That
-produces exactly one run with every test in it.
-
-```yaml
-test:
-  strategy:
-    matrix:
-      shard: [1, 2, 3, 4]
-  steps:
-    - uses: actions/checkout@v4
-    - run: npx playwright test --shard=${{ matrix.shard }}/4 --reporter=blob
-    - uses: actions/upload-artifact@v4
-      with:
-        name: blob-${{ matrix.shard }}
-        path: blob-report
-
-merge:
-  needs: test
-  steps:
-    # Required - see the note below.
-    - uses: actions/checkout@v4
-    - uses: actions/download-artifact@v4
-      with:
-        pattern: blob-*
-        merge-multiple: true
-        path: all-blobs
-    - run: npx playwright merge-reports --reporter=squally-reporter ./all-blobs
-      env:
-        SQUALLY_API_URL: ${{ secrets.SQUALLY_API_URL }}
-        SQUALLY_API_KEY: ${{ secrets.SQUALLY_API_KEY }}
-```
-
-**The merge job needs the repository checked out.** Branch, commit, commit
-message and author are read on whichever machine the reporter runs on - during
-a merge that is the merge job, not the shards. Without a checkout (and without
-the CI provider's own commit variables) they all come out empty, and a run
-without a branch is never counted as a run on your stable branch, which is what
-Squally's broken/healthy verdicts are based on. `actions/checkout` in the merge
-job is enough.
-
-The run's start time is taken from the earliest test, not from when the
-reporter started, so a merge that happens minutes after the tests still dates
-the run correctly.
+**Handled automatically - no extra setup needed.** Each shard is still a
+separate process with its own reporter instance, but Squally now correlates
+them itself: all shards of one CI run are grouped together and shown as one
+run once every shard has finished, with the flip side of that guarantee being
+that the run only completes once every shard *has* reported.
+
+Correlation is based on an id read from your CI provider's own environment,
+so it costs you nothing on GitHub Actions, GitLab CI or CircleCI. If your
+provider is not one of those (or you shard locally across several separate
+`--shard` invocations), set `SQUALLY_CI_RUN_ID` to the same value for every
+shard of one run - anything unique per CI run and shared by its shards works,
+e.g. a build number your CI system already exposes.
+
+**`playwright merge-reports` is not part of this and is not recommended** for
+getting a single run in Squally - the automatic correlation above is the
+supported path, and produces correct per-shard timing/branch/commit
+information that a merge job cannot (see the historical note below). Nothing
+stops you from using `merge-reports` for Playwright's own HTML report if you
+want that separately; just don't point this reporter at the merged result
+expecting Squally to treat it specially.
+
+<details>
+<summary>Why not just use `merge-reports`, historically</summary>
+
+An earlier version of this reporter recommended `playwright merge-reports` as
+the way to get one Squally run out of a sharded job. That path has a real
+problem: branch, commit, commit message and author are all read on whichever
+machine the reporter actually runs on - during a merge that is the merge job,
+not the shards - and without a checkout there (and without the CI provider's
+own commit variables) they all come out empty. A run with no branch is never
+counted as a run on your stable branch, which is what Squally's broken/healthy
+verdicts are based on - so a "merged" run could exist in the dashboard while
+silently contributing nothing to flakiness detection. The automatic
+correlation above reads that information from each shard's own environment
+instead, so this problem does not arise.
+
+</details>
 
 ## Tested revision (optional)
 

package/dist/index.d.ts

@@ -1,22 +1,111 @@
-import type { FullConfig, FullResult, Reporter, Suite, TestCase, TestResult } from "@playwright/test/reporter";
+import type { FullConfig, FullResult, Reporter, Suite, TestCase, TestError, TestResult } from "@playwright/test/reporter";
 interface SquallyReporterOptions {
     apiUrl?: string;
     apiKey?: string;
 }
+interface AppendErrorEntry {
+    message: string;
+    stack?: string;
+    snippet?: string;
+    location?: {
+        file: string;
+        line: number;
+        column: number;
+    };
+}
+interface AppendResultEntry {
+    testName: string;
+    filePath: string | null;
+    status: "passed" | "failed" | "skipped";
+    retryCount: number;
+    repeatIndex: number;
+    durationMs: number | null;
+    errors?: AppendErrorEntry[];
+    consoleOutput: string | null;
+    tags?: string[];
+    browserName: string | null;
+    errorContext: string | null;
+}
+type ArtifactType = "screenshot" | "trace";
+declare function truncate(value: string, max: number): string;
+declare function truncateOrNull(value: string | null, max: number): string | null;
+declare function mapStatus(status: TestResult["status"]): AppendResultEntry["status"];
+declare function detectCiRunId(env: NodeJS.ProcessEnv, generateFallback: () => string): string;
+interface ProjectSuiteLike {
+    project(): {
+        name: string;
+    } | undefined;
+}
+declare function resolveProjectName(projectSuites: readonly ProjectSuiteLike[]): string | null;
+declare function normalizeRevision(raw: string): string;
+interface ResolvedRevision {
+    revision: string;
+    source: "ci_native" | "passed_param";
+}
+declare function resolveTestedRevision(): ResolvedRevision | null;
+declare function toPosixPath(p: string): string;
+declare function stripAnsi(value: string): string;
+declare function maskSecrets(value: string): string;
+declare function prepareConsoleOutput(value: string | null): string | null;
+declare function mapErrors(errors: readonly TestError[]): AppendErrorEntry[] | undefined;
+declare function attemptQualifiesForArtifacts(mappedStatus: AppendResultEntry["status"], retryCount: number): boolean;
+declare function artifactKey(testName: string, filePath: string, retryCount: number, type: ArtifactType): string;
+declare function parseRetryAfterMs(headerValue: string | null, nowMs: number): number | null;
+declare function computeBackoffMs(retryNumber: number): number;
 export default class SquallyReporter implements Reporter {
     private apiUrl;
     private apiKey;
     private enabled;
     private startedAt;
-    private results;
+    private dispatcher;
+    private ciRunId;
+    private shardIndex;
+    private shardTotal;
+    private projectName;
+    private runTags;
+    private runConfig;
+    private buffer;
     private artifacts;
-    private earliestTestStart;
+    private sendingDisabled;
+    private flushTimerHandle;
+    private inFlightFlushes;
+    private runIdPromise;
+    private runId;
     constructor(options?: SquallyReporterOptions);
-    onBegin(config: FullConfig, _suite: Suite): void;
+    onBegin(config: FullConfig, suite: Suite): void;
+    private resolveRunConfig;
     onTestEnd(test: TestCase, result: TestResult): void;
-    onEnd(result: FullResult): Promise<void>;
-    private runStartedAt;
+    onEnd: (result: FullResult) => Promise<void>;
+    private drainAndComplete;
+    private armFlushTimer;
+    private stopFlushTimer;
+    private giveUp;
+    private triggerFlush;
+    private flush;
+    private sendBatch;
+    private beginRun;
+    private completeRun;
+    private fetchRules;
+    private requestJson;
     private uploadArtifacts;
     private presignAndUpload;
 }
+export declare const __internal: {
+    truncate: typeof truncate;
+    truncateOrNull: typeof truncateOrNull;
+    mapStatus: typeof mapStatus;
+    stripAnsi: typeof stripAnsi;
+    normalizeRevision: typeof normalizeRevision;
+    resolveTestedRevision: typeof resolveTestedRevision;
+    toPosixPath: typeof toPosixPath;
+    detectCiRunId: typeof detectCiRunId;
+    resolveProjectName: typeof resolveProjectName;
+    maskSecrets: typeof maskSecrets;
+    prepareConsoleOutput: typeof prepareConsoleOutput;
+    mapErrors: typeof mapErrors;
+    parseRetryAfterMs: typeof parseRetryAfterMs;
+    computeBackoffMs: typeof computeBackoffMs;
+    attemptQualifiesForArtifacts: typeof attemptQualifiesForArtifacts;
+    artifactKey: typeof artifactKey;
+};
 export {};