squally-reporter 0.5.1 → 0.6.0

package/README.md

@@ -110,6 +110,13 @@ If Squally is unreachable partway through, results already sent stay in the
 dashboard; only what could not be delivered is lost, and your test run's exit
 code is never affected either way.
 
+**Only actual test runs appear.** A run shows up in Squally once your first
+test starts, so commands that never run a test create nothing: `--list`,
+a `globalSetup` or `webServer` that fails to start, a `--grep` that matches
+nothing. A run that *does* execute and legitimately contains no tests — an
+empty shard, `--only-changed` with no changed test file, `--pass-with-no-tests`
+— is still reported, as a run with zero results.
+
 ## Sharding
 
 **Handled automatically - no extra setup needed.** Each shard is still a

package/dist/index.d.ts

@@ -29,6 +29,18 @@ interface AppendResultEntry {
 type ArtifactType = "screenshot" | "trace";
 declare function truncate(value: string, max: number): string;
 declare function truncateOrNull(value: string | null, max: number): string | null;
+/**
+ * This shard's flush interval: the base cadence stretched by how many shards
+ * share the rate budget with it.
+ *
+ * Only the TIMER is scaled. FLUSH_THRESHOLD_RESULTS is not, and that is
+ * deliberate: it is a latency bound on a fast suite, not a rate control, and
+ * stretching it would delay results without bounding the burst any better -
+ * the burst is bounded by how fast tests finish, which is not ours to set.
+ * The spec's acknowledged worst case (a very fast unit-style suite crossing 25
+ * faster than the interval) is therefore unchanged by this.
+ */
+declare function flushIntervalFor(shardTotal: number | null): number;
 declare function mapStatus(status: TestResult["status"]): AppendResultEntry["status"];
 declare function detectCiRunId(env: NodeJS.ProcessEnv, generateFallback: () => string): string;
 interface ProjectSuiteLike {
@@ -66,13 +78,51 @@ export default class SquallyReporter implements Reporter {
     private runConfig;
     private buffer;
     private artifacts;
+    /**
+     * Every result this shard produced, counted where it is produced (0.6.0).
+     *
+     * SIX PATHS LOSE RESULTS, and this counter exists because five of them are
+     * invisible afterwards and the sixth cannot be counted at all where it
+     * happens:
+     *
+     *   1. a batch dropped after its retries are spent (429, 5xx, network,
+     *      unparseable response) - sendBatch's default branch
+     *   2. a batch dropped as malformed (400), which is not retried
+     *   3. the whole unsent buffer discarded by giveUp() on 402/404/409
+     *   4. every result produced AFTER a give-up - onTestEnd returns before the
+     *      buffer, so there is nothing to count at the point of loss
+     *   5. batches still in flight when the drain budget expires at onEnd; the
+     *      process then completes the run and exits out from under them
+     *   6. onBegin never getting a runId, so nothing was ever sendable
+     *
+     * A drop counter would have to be correct in all six places forever, and
+     * path 4 has no place to put it. Counting production instead needs one
+     * increment in one function, and it makes loss visible even on a path this
+     * reporter does not yet know about - the server compares this against what
+     * it actually stored.
+     *
+     * Per shard, for this shard's runId: each shard reports its own production
+     * against its own row, so no group arithmetic is needed on either side.
+     */
+    private resultsProduced;
     private sendingDisabled;
     private flushTimerHandle;
     private inFlightFlushes;
+    private runStarted;
     private runIdPromise;
     private runId;
     constructor(options?: SquallyReporterOptions);
     onBegin(config: FullConfig, suite: Suite): void;
+    /**
+     * Create the run server-side. Idempotent, and deliberately not called from
+     * onBegin.
+     *
+     * The flush timer is armed here rather than in onBegin for the same reason:
+     * nothing can be in the buffer before a test has ended, and a timer that can
+     * only ever fire on an empty buffer is a timer with nothing to do.
+     */
+    private startRun;
+    onTestBegin(): void;
     private resolveRunConfig;
     onTestEnd(test: TestCase, result: TestResult): void;
     onEnd: (result: FullResult) => Promise<void>;
@@ -105,6 +155,7 @@ export declare const __internal: {
     mapErrors: typeof mapErrors;
     parseRetryAfterMs: typeof parseRetryAfterMs;
     computeBackoffMs: typeof computeBackoffMs;
+    flushIntervalFor: typeof flushIntervalFor;
     attemptQualifiesForArtifacts: typeof attemptQualifiesForArtifacts;
     artifactKey: typeof artifactKey;
 };

package/dist/index.js

@@ -138,6 +138,47 @@ const RULES_REQUEST_TIMEOUT_MS = 5000;
 // API key): 60 s / 2 s = 30 requests/minute, half of 60.
 const FLUSH_THRESHOLD_RESULTS = 25;
 const FLUSH_INTERVAL_MS = 2000;
+// Scaled by the shard count (0.6.0), because the budget the 2 s above was
+// derived from is NOT per reporter - it is per API key, and every shard of a
+// run carries the same key. The firewall counts `header:authorization`, so
+// eight shards each flushing every 2 s produce 8 x 30 = 240 requests/minute
+// against a 60/minute rule. The "half of 60" margin the constant above claims
+// was computed for a single reporter and silently stopped holding the day
+// sharding arrived.
+//
+// interval x shards keeps the AGGREGATE rate at 30/minute for any shard count:
+// S shards x (60 s / 2S s) = 30. The live view does not get slower for it
+// either - it shows the RUN, and a run made of S shards receives an update
+// from one shard or another every 2 s on average regardless of S.
+//
+// CAPPED, and the cap is about crash exposure rather than rate. Everything
+// buffered is lost if the runner is killed (CI cancellation, OOM), so the
+// interval is also the size of the window at risk. 30 s is the same figure
+// RATE_LIMIT_BACKOFF_CAP_MS uses for "the longest we ever wait", and it keeps
+// the aggregate inside the budget up to 30 shards (S x 60/30 <= 60), which
+// covers the 4-to-16 range the app's own shard documentation describes with
+// room to spare.
+//
+// Above 30 shards the aggregate creeps over the rule again. That is accepted:
+// it is outside the documented range, the retry path already handles a 429,
+// and since 0.6.0 any batch actually lost to it is visible in the
+// resultsProduced diff rather than silent.
+const FLUSH_INTERVAL_CAP_MS = 30000;
+/**
+ * This shard's flush interval: the base cadence stretched by how many shards
+ * share the rate budget with it.
+ *
+ * Only the TIMER is scaled. FLUSH_THRESHOLD_RESULTS is not, and that is
+ * deliberate: it is a latency bound on a fast suite, not a rate control, and
+ * stretching it would delay results without bounding the burst any better -
+ * the burst is bounded by how fast tests finish, which is not ours to set.
+ * The spec's acknowledged worst case (a very fast unit-style suite crossing 25
+ * faster than the interval) is therefore unchanged by this.
+ */
+function flushIntervalFor(shardTotal) {
+    const shards = Math.max(1, shardTotal ?? 1);
+    return Math.min(FLUSH_INTERVAL_CAP_MS, FLUSH_INTERVAL_MS * shards);
+}
 // --- Retry policy (spec section 3.2) ----------------------------------------
 //
 // Two independent policies, both applied by requestJson() below via separate
@@ -295,6 +336,39 @@ function detectAuthorName() {
 // the same escape-hatch pattern this file already uses for
 // SAME_REPO_AS_APP/TESTED_REVISION below.
 //
+// Is this invocation `playwright test --list` - "collect all the tests and
+// report them, but do not run" (the flag's own help text)?
+//
+// WHY THIS IS AN OPTIMISATION AND NOT THE MECHANISM. A list run is a customer
+// enumerating tests: in CI, in an editor, or from this repo's own AGENTS.md,
+// which cites `--list` as a check. It is not a test run, and until 0.5.2 the
+// reporter created one anyway - a completed run with zero results, consuming a
+// run number and a slot in the customer's monthly quota.
+//
+// MEASURED, both directions, on 1.62.1 and on 1.30.0 (this package's peer
+// floor):
+//   - `config.argv` is documented ("Snapshot of process.argv captured in the
+//     runner process") and carries `--list` on 1.62.1. On 1.30.0 the field does
+//     not exist at all - not in the types, not on the object. So it cannot be
+//     the mechanism for a package whose peer range starts at 1.30.
+//   - `process.argv` carries `--list` on BOTH, because a reporter is loaded in
+//     the runner process, which is the CLI process. Hence the fallback.
+//   - There is no `listOnly` on FullConfig on either version. The only other
+//     difference between a list run and a real one is behavioural: onBegin
+//     fires with the suite fully populated, and then no test ever begins.
+//
+// That behavioural fingerprint is what the run-creation deferral in the class
+// below relies on, and it is why THIS function is allowed to be imperfect: an
+// invocation that lists tests without `--list` on its argv - an IDE driving the
+// runner through its own entry point, say - is still covered, because it also
+// never begins a test. What this function buys on top is silence: a list run
+// makes no network call at all, not even the rules GET, which matters when an
+// editor lists tests on every keystroke-triggered reload.
+function isListOnlyInvocation(config) {
+    const configArgv = config.argv;
+    const argv = Array.isArray(configArgv) ? configArgv : process.argv;
+    return argv.some((arg) => arg === "--list");
+}
 // env and generateFallback are parameters, not process.env/randomUUID read
 // directly, so this is unit-testable without mocking global state.
 function detectCiRunId(env, generateFallback) {
@@ -739,6 +813,33 @@ class SquallyReporter {
         this.projectName = null;
         this.buffer = [];
         this.artifacts = [];
+        /**
+         * Every result this shard produced, counted where it is produced (0.6.0).
+         *
+         * SIX PATHS LOSE RESULTS, and this counter exists because five of them are
+         * invisible afterwards and the sixth cannot be counted at all where it
+         * happens:
+         *
+         *   1. a batch dropped after its retries are spent (429, 5xx, network,
+         *      unparseable response) - sendBatch's default branch
+         *   2. a batch dropped as malformed (400), which is not retried
+         *   3. the whole unsent buffer discarded by giveUp() on 402/404/409
+         *   4. every result produced AFTER a give-up - onTestEnd returns before the
+         *      buffer, so there is nothing to count at the point of loss
+         *   5. batches still in flight when the drain budget expires at onEnd; the
+         *      process then completes the run and exits out from under them
+         *   6. onBegin never getting a runId, so nothing was ever sendable
+         *
+         * A drop counter would have to be correct in all six places forever, and
+         * path 4 has no place to put it. Counting production instead needs one
+         * increment in one function, and it makes loss visible even on a path this
+         * reporter does not yet know about - the server compares this against what
+         * it actually stored.
+         *
+         * Per shard, for this shard's runId: each shard reports its own production
+         * against its own row, so no group arithmetic is needed on either side.
+         */
+        this.resultsProduced = 0;
         // True once nothing more will ever be sent for this run - a permanent
         // give-up (402/404/409 on append, or onBegin never getting a runId at
         // all). onTestEnd stops collecting further results/artifacts once this is
@@ -746,9 +847,15 @@ class SquallyReporter {
         this.sendingDisabled = false;
         this.flushTimerHandle = null;
         this.inFlightFlushes = new Set();
-        // Resolves to the runId once onBegin succeeds, or to null on any permanent
+        // Has the run been created server-side? Set by startRun(), which fires on
+        // the first test rather than in onBegin (0.5.2). While this is false, this
+        // process has sent nothing that could become a run in the dashboard.
+        this.runStarted = false;
+        // Resolves to the runId once beginRun() succeeds, or to null on any permanent
         // failure (never rejects - every failure path here is a warning, not an
-        // exception, same principle as the rest of this file).
+        // exception, same principle as the rest of this file). Stays the initial
+        // resolved-null promise until startRun() replaces it, which is what makes
+        // flush() a no-op for an invocation that never ran a test.
         this.runIdPromise = Promise.resolve(null);
         // Set once runIdPromise resolves to a real id - read by onEnd's artifact
         // upload step, which needs a plain value rather than a promise it would
@@ -814,6 +921,15 @@ class SquallyReporter {
     onBegin(config, suite) {
         if (!this.enabled)
             return;
+        // A list run is not a run. Disabling the reporter outright is what makes
+        // every later hook - and onEnd's dispatcher.close() - a no-op, so nothing
+        // below this line has to know about the case. Silent on purpose: `--list`
+        // output is parsed by tooling, and a warning printed on every editor
+        // refresh would be noise about a decision the customer did not make.
+        if (isListOnlyInvocation(config)) {
+            this.enabled = false;
+            return;
+        }
         this.startedAt = new Date();
         this.dispatcher = new undici_1.Agent({
             keepAliveTimeout: 4000,
@@ -837,10 +953,44 @@ class SquallyReporter {
         this.runTags =
             tags.length > 0 ? tags.slice(0, MAX_TAGS).map((tag) => truncate(tag, V2_FIELD_LIMITS.tag)) : undefined;
         this.runConfig = this.resolveRunConfig(config);
-        this.runIdPromise = this.beginRun();
+        // NO beginRun() HERE ANY MORE (0.5.2). Until 0.5.1 this line created the
+        // run server-side the moment Playwright started, which meant every
+        // invocation that never executed a test still produced one - measured
+        // against a logging mock: `--list`, a failing globalSetup and a webServer
+        // that will not start all reach onBegin and all created a completed,
+        // empty run. The run is now created by startRun() when the first test
+        // actually begins, and by drainAndComplete() for the one case where a run
+        // that executed nothing must still be reported (see there).
+        //
+        // The rules GET stays here: it has to be in flight BEFORE tests run to be
+        // of any use, it creates nothing, and it costs no quota.
         void this.fetchRules();
+    }
+    /**
+     * Create the run server-side. Idempotent, and deliberately not called from
+     * onBegin.
+     *
+     * The flush timer is armed here rather than in onBegin for the same reason:
+     * nothing can be in the buffer before a test has ended, and a timer that can
+     * only ever fire on an empty buffer is a timer with nothing to do.
+     */
+    startRun() {
+        if (this.runStarted || !this.enabled)
+            return;
+        this.runStarted = true;
+        this.runIdPromise = this.beginRun();
         this.armFlushTimer();
     }
+    // The trigger. onTestBegin fires for every test Playwright actually
+    // executes, INCLUDING skipped ones - measured, both the statically skipped
+    // (`test.skip("name", fn)`) and the conditionally skipped
+    // (`test.skip(true, "reason")`) form - so a suite whose tests are all
+    // skipped is still a run, and still reported.
+    onTestBegin() {
+        if (!this.enabled)
+            return;
+        this.startRun();
+    }
     // config.browsers ({name, version}[]) is deliberately OMITTED here, not
     // sent as an empty/placeholder array. Getting REAL installed browser
     // versions would mean either launching each browser type just to query it
@@ -863,8 +1013,28 @@ class SquallyReporter {
         };
     }
     onTestEnd(test, result) {
-        if (!this.enabled || this.sendingDisabled)
+        // Two guards, deliberately split by the counter between them.
+        //
+        // `enabled` is false when there is nothing to report to at all (no URL or
+        // key, or `--list`): no run exists, no /complete will be sent, and there
+        // is nothing for a count to be compared against.
+        //
+        // `sendingDisabled` is different - a run DOES exist server-side, it will
+        // be completed, and every result arriving after the give-up is lost. That
+        // is loss path 4, and counting BEFORE this return is the only place it can
+        // be seen. Moving the increment below the return would make the largest
+        // loss path of the six silently report zero.
+        if (!this.enabled)
+            return;
+        this.resultsProduced += 1;
+        if (this.sendingDisabled)
             return;
+        // Belt and braces: onTestBegin has already done this for every test
+        // Playwright executes, but a result arriving without its begin would
+        // otherwise be buffered against a run that was never created and dropped
+        // silently at onEnd. startRun() is idempotent, so the ordinary path pays
+        // one boolean check.
+        this.startRun();
         // Truncated HERE, once, not when the payload is built: testName/filePath
         // are Squally's test identity, and /api/artifacts/presign resolves each
         // artifact back to a TestResult through exactly these two values plus
@@ -955,17 +1125,51 @@ class SquallyReporter {
     // (deliberate divergence from v1) Once a runId exists, /complete is ALWAYS
     // called - there is no "this run had zero results, skip reporting it
     // entirely" special case, unlike v1 (`if (this.results.length === 0)
-    // return;`, guarding the whole of onEnd there). That check cannot be
+    // return;`, guarding the whole of onEnd there). ~~That check cannot be
     // reproduced here: onBegin already created the run server-side, before a
-    // single test could have told us the eventual count would be zero.
-    // Skipping /complete at this point would not undo the run's existence - it
-    // would leave an already-created run stuck in `running` until the timeout
-    // sweep notices (currently up to 18 hours, see docs/status.md's run-timeout
-    // entry). Completing it immediately with zero results is strictly better: a
-    // short-lived empty run in the dashboard costs less than an
-    // apparently-stuck one for the better part of a day.
+    // single test could have told us the eventual count would be zero.~~
+    // CORRECTED 0.5.2: onBegin no longer creates it, so the decision IS
+    // available here - and it is taken by the block at the top of the method
+    // below, which is a stricter version of v1's check. What stands unchanged is
+    // the reasoning about a run that HAS been created: skipping /complete for it
+    // would not undo its existence, it would leave it stuck in `running` until
+    // the timeout sweep notices (up to 18 hours, see docs/status.md's run-timeout
+    // entry).
     async drainAndComplete(result) {
         this.stopFlushTimer();
+        // NOTHING RAN. Two cases that look identical from here and must not be
+        // treated identically (0.5.2):
+        //
+        //   1. The suite the customer asked for was EMPTY, and Playwright ran it
+        //      to completion anyway. `--pass-with-no-tests`, `--only-changed` on a
+        //      commit that touched no test file, and - the one that matters most -
+        //      a shard with nothing in it, which happens whenever there are fewer
+        //      tests than shards. All three exit 0 with FullResult.status
+        //      "passed", measured. These MUST still be reported: a customer who
+        //      filters to a tag with no tests and sees nothing at all concludes
+        //      the reporter is broken, and an unreported empty shard is worse than
+        //      that - its group would never reach its shard total and would sit
+        //      incomplete forever (squally-app docs/shard-ui-spec.md section 5.5).
+        //
+        //   2. Playwright never got as far as running the suite: a globalSetup
+        //      that threw, a webServer that would not start, `--grep` matching
+        //      nothing without `--pass-with-no-tests`, `--last-failed` with
+        //      nothing to repeat. All measured: onBegin fires, an onError follows,
+        //      and FullResult.status is "failed" - the run ends in an error state
+        //      having executed nothing. Reporting these produced an empty run
+        //      whose only content was that it was empty, at the cost of a run
+        //      number and a quota slot.
+        //
+        // The line between them is FullResult.status, and that is the whole rule:
+        // Squally reports a run when Playwright ran the suite to completion,
+        // whatever the suite turned out to contain, and does not when Playwright
+        // never got that far. `--list` never reaches here at all - onBegin
+        // disables the reporter outright.
+        if (!this.runStarted) {
+            if (result.status !== "passed")
+                return;
+            this.startRun();
+        }
         const deadline = Date.now() + DRAIN_BUDGET_MS;
         const runId = await withDeadline(this.runIdPromise, deadline);
         if (runId === TIMED_OUT) {
@@ -992,7 +1196,11 @@ class SquallyReporter {
         const handle = setTimeout(() => {
             this.triggerFlush();
             this.armFlushTimer();
-        }, FLUSH_INTERVAL_MS);
+            // Read per arming rather than cached at onBegin: shardTotal is known
+            // before the first arming (config.shard.total, set in onBegin), so this
+            // is the same value every time - reading it here keeps the timer honest
+            // if that ever stops being true.
+        }, flushIntervalFor(this.shardTotal));
         handle.unref();
         this.flushTimerHandle = handle;
     }
@@ -1132,6 +1340,9 @@ class SquallyReporter {
             shardIndex: this.shardIndex,
             status,
             finishedAt: new Date().toISOString(),
+            // Read at the last possible moment: every loss path has already happened
+            // by the time /complete is sent, so this number is final.
+            resultsProduced: this.resultsProduced,
         };
         const outcome = await this.requestJson("POST", `${this.apiUrl}/api/ingest/v2/runs/${runId}/complete`, payload);
         if (outcome.kind !== "ok") {
@@ -1393,6 +1604,7 @@ exports.__internal = {
     mapErrors,
     parseRetryAfterMs,
     computeBackoffMs,
+    flushIntervalFor,
     attemptQualifiesForArtifacts,
     artifactKey,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squally-reporter",
-  "version": "0.5.1",
+  "version": "0.6.0",
   "description": "Playwright reporter that sends test results to your Squally dashboard.",
   "author": "Squally <squally.dev@gmail.com>",
   "keywords": [
@@ -24,7 +24,8 @@
     "prepare": "npm run build",
     "test:unit": "node --test \"test/unit/**/*.test.js\"",
     "test:exit-code": "node test/exit-code/run.mjs",
-    "test": "npm run build && npm run test:unit && npm run test:exit-code"
+    "test:empty-runs": "node test/empty-runs/run.mjs",
+    "test": "npm run build && npm run test:unit && npm run test:exit-code && npm run test:empty-runs"
   },
   "peerDependencies": {
     "@playwright/test": ">=1.30.0"