squally-reporter 0.5.0 → 0.5.2

package/README.md

@@ -110,6 +110,13 @@ If Squally is unreachable partway through, results already sent stay in the
 dashboard; only what could not be delivered is lost, and your test run's exit
 code is never affected either way.
 
+**Only actual test runs appear.** A run shows up in Squally once your first
+test starts, so commands that never run a test create nothing: `--list`,
+a `globalSetup` or `webServer` that fails to start, a `--grep` that matches
+nothing. A run that *does* execute and legitimately contains no tests — an
+empty shard, `--only-changed` with no changed test file, `--pass-with-no-tests`
+— is still reported, as a run with zero results.
+
 ## Sharding
 
 **Handled automatically - no extra setup needed.** Each shard is still a

package/dist/index.d.ts

@@ -69,10 +69,21 @@ export default class SquallyReporter implements Reporter {
     private sendingDisabled;
     private flushTimerHandle;
     private inFlightFlushes;
+    private runStarted;
     private runIdPromise;
     private runId;
     constructor(options?: SquallyReporterOptions);
     onBegin(config: FullConfig, suite: Suite): void;
+    /**
+     * Create the run server-side. Idempotent, and deliberately not called from
+     * onBegin.
+     *
+     * The flush timer is armed here rather than in onBegin for the same reason:
+     * nothing can be in the buffer before a test has ended, and a timer that can
+     * only ever fire on an empty buffer is a timer with nothing to do.
+     */
+    private startRun;
+    onTestBegin(): void;
     private resolveRunConfig;
     onTestEnd(test: TestCase, result: TestResult): void;
     onEnd: (result: FullResult) => Promise<void>;

package/dist/index.js

@@ -295,6 +295,39 @@ function detectAuthorName() {
 // the same escape-hatch pattern this file already uses for
 // SAME_REPO_AS_APP/TESTED_REVISION below.
 //
+// Is this invocation `playwright test --list` - "collect all the tests and
+// report them, but do not run" (the flag's own help text)?
+//
+// WHY THIS IS AN OPTIMISATION AND NOT THE MECHANISM. A list run is a customer
+// enumerating tests: in CI, in an editor, or from this repo's own AGENTS.md,
+// which cites `--list` as a check. It is not a test run, and until 0.5.2 the
+// reporter created one anyway - a completed run with zero results, consuming a
+// run number and a slot in the customer's monthly quota.
+//
+// MEASURED, both directions, on 1.62.1 and on 1.30.0 (this package's peer
+// floor):
+//   - `config.argv` is documented ("Snapshot of process.argv captured in the
+//     runner process") and carries `--list` on 1.62.1. On 1.30.0 the field does
+//     not exist at all - not in the types, not on the object. So it cannot be
+//     the mechanism for a package whose peer range starts at 1.30.
+//   - `process.argv` carries `--list` on BOTH, because a reporter is loaded in
+//     the runner process, which is the CLI process. Hence the fallback.
+//   - There is no `listOnly` on FullConfig on either version. The only other
+//     difference between a list run and a real one is behavioural: onBegin
+//     fires with the suite fully populated, and then no test ever begins.
+//
+// That behavioural fingerprint is what the run-creation deferral in the class
+// below relies on, and it is why THIS function is allowed to be imperfect: an
+// invocation that lists tests without `--list` on its argv - an IDE driving the
+// runner through its own entry point, say - is still covered, because it also
+// never begins a test. What this function buys on top is silence: a list run
+// makes no network call at all, not even the rules GET, which matters when an
+// editor lists tests on every keystroke-triggered reload.
+function isListOnlyInvocation(config) {
+    const configArgv = config.argv;
+    const argv = Array.isArray(configArgv) ? configArgv : process.argv;
+    return argv.some((arg) => arg === "--list");
+}
 // env and generateFallback are parameters, not process.env/randomUUID read
 // directly, so this is unit-testable without mocking global state.
 function detectCiRunId(env, generateFallback) {
@@ -640,10 +673,45 @@ function confirmReasonMessage(reason) {
 function artifactLabel(type) {
     return type === "screenshot" ? "Screenshot" : "Trace";
 }
+/**
+ * Wait, WITHOUT letting the process die during the wait.
+ *
+ * The timer is deliberately NOT unref'd, and that is the whole content of this
+ * function. An unref'd timer does not hold Node's event loop open, so a promise
+ * that only an unref'd timer can resolve is a promise the process may never
+ * wait for: if nothing else is pending, Node treats the loop as empty and exits
+ * THERE, with whatever `process.exitCode` happens to be - which is 0.
+ *
+ * That shipped in 0.5.0 and made a FAILING test suite report a GREEN CI job:
+ *
+ *   0.4.0  await new Promise((resolve) => setTimeout(resolve, EXIT_DRAIN_MS));
+ *   0.5.0  await delay(EXIT_DRAIN_MS);   // helper below, which unref'd
+ *
+ * Playwright computes the exit code only AFTER awaiting every reporter's
+ * onEnd - the runner multiplexer awaits `reporter.onEnd(result)` and the CLI
+ * then calls `gracefullyProcessExitDoNotHang(status === "passed" ? 0 : 1)`.
+ * Exiting during onEnd skips the line that would have set 1. Measured: the same
+ * 250 ms wait exits 0 when unref'd and 1 when not, on a suite with a failing
+ * test.
+ *
+ * EVERY caller is awaited from inside onEnd, directly or through
+ * drainAndComplete's retries, so every one of them needs this. There is no
+ * "this wait is unimportant" case: a wait nobody needs should not be awaited at
+ * all. The first fix for the bug above kept an opt-out for the retry backoffs
+ * and was still wrong - a server returning 503 on /complete reproduced the
+ * green-on-red exit through exactly that path (measured, 10.09.).
+ *
+ * Ref'ing costs nothing here: Playwright ends the process with `process.exit()`,
+ * which does not wait for pending timers, so a still-running backoff cannot
+ * delay anyone's CI.
+ *
+ * NOT to be confused with the two unref'd timers elsewhere in this file, both
+ * of which are correct: withDeadline's timer races a real promise, and the
+ * periodic flush timer is never awaited by anything.
+ */
 function delay(ms) {
     return new Promise((resolve) => {
-        const handle = setTimeout(resolve, ms);
-        handle.unref();
+        setTimeout(resolve, ms);
     });
 }
 // Retry-After per RFC 9110: either an integer number of seconds, or an
@@ -711,9 +779,15 @@ class SquallyReporter {
         this.sendingDisabled = false;
         this.flushTimerHandle = null;
         this.inFlightFlushes = new Set();
-        // Resolves to the runId once onBegin succeeds, or to null on any permanent
+        // Has the run been created server-side? Set by startRun(), which fires on
+        // the first test rather than in onBegin (0.5.2). While this is false, this
+        // process has sent nothing that could become a run in the dashboard.
+        this.runStarted = false;
+        // Resolves to the runId once beginRun() succeeds, or to null on any permanent
         // failure (never rejects - every failure path here is a warning, not an
-        // exception, same principle as the rest of this file).
+        // exception, same principle as the rest of this file). Stays the initial
+        // resolved-null promise until startRun() replaces it, which is what makes
+        // flush() a no-op for an invocation that never ran a test.
         this.runIdPromise = Promise.resolve(null);
         // Set once runIdPromise resolves to a real id - read by onEnd's artifact
         // upload step, which needs a plain value rather than a promise it would
@@ -754,6 +828,9 @@ class SquallyReporter {
                 // See EXIT_DRAIN_MS - closing the dispatcher resolves before the
                 // underlying sockets are fully gone, and Playwright exits the process
                 // the moment onEnd() resolves.
+                // This wait exists precisely to keep the process alive while sockets
+                // finish closing - see delay(), and test/exit-code's failing-suite
+                // scenario, which is what catches it if the unref ever comes back.
                 await delay(EXIT_DRAIN_MS);
             }
         };
@@ -776,6 +853,15 @@ class SquallyReporter {
     onBegin(config, suite) {
         if (!this.enabled)
             return;
+        // A list run is not a run. Disabling the reporter outright is what makes
+        // every later hook - and onEnd's dispatcher.close() - a no-op, so nothing
+        // below this line has to know about the case. Silent on purpose: `--list`
+        // output is parsed by tooling, and a warning printed on every editor
+        // refresh would be noise about a decision the customer did not make.
+        if (isListOnlyInvocation(config)) {
+            this.enabled = false;
+            return;
+        }
         this.startedAt = new Date();
         this.dispatcher = new undici_1.Agent({
             keepAliveTimeout: 4000,
@@ -799,10 +885,44 @@ class SquallyReporter {
         this.runTags =
             tags.length > 0 ? tags.slice(0, MAX_TAGS).map((tag) => truncate(tag, V2_FIELD_LIMITS.tag)) : undefined;
         this.runConfig = this.resolveRunConfig(config);
-        this.runIdPromise = this.beginRun();
+        // NO beginRun() HERE ANY MORE (0.5.2). Until 0.5.1 this line created the
+        // run server-side the moment Playwright started, which meant every
+        // invocation that never executed a test still produced one - measured
+        // against a logging mock: `--list`, a failing globalSetup and a webServer
+        // that will not start all reach onBegin and all created a completed,
+        // empty run. The run is now created by startRun() when the first test
+        // actually begins, and by drainAndComplete() for the one case where a run
+        // that executed nothing must still be reported (see there).
+        //
+        // The rules GET stays here: it has to be in flight BEFORE tests run to be
+        // of any use, it creates nothing, and it costs no quota.
         void this.fetchRules();
+    }
+    /**
+     * Create the run server-side. Idempotent, and deliberately not called from
+     * onBegin.
+     *
+     * The flush timer is armed here rather than in onBegin for the same reason:
+     * nothing can be in the buffer before a test has ended, and a timer that can
+     * only ever fire on an empty buffer is a timer with nothing to do.
+     */
+    startRun() {
+        if (this.runStarted || !this.enabled)
+            return;
+        this.runStarted = true;
+        this.runIdPromise = this.beginRun();
         this.armFlushTimer();
     }
+    // The trigger. onTestBegin fires for every test Playwright actually
+    // executes, INCLUDING skipped ones - measured, both the statically skipped
+    // (`test.skip("name", fn)`) and the conditionally skipped
+    // (`test.skip(true, "reason")`) form - so a suite whose tests are all
+    // skipped is still a run, and still reported.
+    onTestBegin() {
+        if (!this.enabled)
+            return;
+        this.startRun();
+    }
     // config.browsers ({name, version}[]) is deliberately OMITTED here, not
     // sent as an empty/placeholder array. Getting REAL installed browser
     // versions would mean either launching each browser type just to query it
@@ -827,6 +947,12 @@ class SquallyReporter {
     onTestEnd(test, result) {
         if (!this.enabled || this.sendingDisabled)
             return;
+        // Belt and braces: onTestBegin has already done this for every test
+        // Playwright executes, but a result arriving without its begin would
+        // otherwise be buffered against a run that was never created and dropped
+        // silently at onEnd. startRun() is idempotent, so the ordinary path pays
+        // one boolean check.
+        this.startRun();
         // Truncated HERE, once, not when the payload is built: testName/filePath
         // are Squally's test identity, and /api/artifacts/presign resolves each
         // artifact back to a TestResult through exactly these two values plus
@@ -917,17 +1043,51 @@ class SquallyReporter {
     // (deliberate divergence from v1) Once a runId exists, /complete is ALWAYS
     // called - there is no "this run had zero results, skip reporting it
     // entirely" special case, unlike v1 (`if (this.results.length === 0)
-    // return;`, guarding the whole of onEnd there). That check cannot be
+    // return;`, guarding the whole of onEnd there). ~~That check cannot be
     // reproduced here: onBegin already created the run server-side, before a
-    // single test could have told us the eventual count would be zero.
-    // Skipping /complete at this point would not undo the run's existence - it
-    // would leave an already-created run stuck in `running` until the timeout
-    // sweep notices (currently up to 18 hours, see docs/status.md's run-timeout
-    // entry). Completing it immediately with zero results is strictly better: a
-    // short-lived empty run in the dashboard costs less than an
-    // apparently-stuck one for the better part of a day.
+    // single test could have told us the eventual count would be zero.~~
+    // CORRECTED 0.5.2: onBegin no longer creates it, so the decision IS
+    // available here - and it is taken by the block at the top of the method
+    // below, which is a stricter version of v1's check. What stands unchanged is
+    // the reasoning about a run that HAS been created: skipping /complete for it
+    // would not undo its existence, it would leave it stuck in `running` until
+    // the timeout sweep notices (up to 18 hours, see docs/status.md's run-timeout
+    // entry).
     async drainAndComplete(result) {
         this.stopFlushTimer();
+        // NOTHING RAN. Two cases that look identical from here and must not be
+        // treated identically (0.5.2):
+        //
+        //   1. The suite the customer asked for was EMPTY, and Playwright ran it
+        //      to completion anyway. `--pass-with-no-tests`, `--only-changed` on a
+        //      commit that touched no test file, and - the one that matters most -
+        //      a shard with nothing in it, which happens whenever there are fewer
+        //      tests than shards. All three exit 0 with FullResult.status
+        //      "passed", measured. These MUST still be reported: a customer who
+        //      filters to a tag with no tests and sees nothing at all concludes
+        //      the reporter is broken, and an unreported empty shard is worse than
+        //      that - its group would never reach its shard total and would sit
+        //      incomplete forever (squally-app docs/shard-ui-spec.md section 5.5).
+        //
+        //   2. Playwright never got as far as running the suite: a globalSetup
+        //      that threw, a webServer that would not start, `--grep` matching
+        //      nothing without `--pass-with-no-tests`, `--last-failed` with
+        //      nothing to repeat. All measured: onBegin fires, an onError follows,
+        //      and FullResult.status is "failed" - the run ends in an error state
+        //      having executed nothing. Reporting these produced an empty run
+        //      whose only content was that it was empty, at the cost of a run
+        //      number and a quota slot.
+        //
+        // The line between them is FullResult.status, and that is the whole rule:
+        // Squally reports a run when Playwright ran the suite to completion,
+        // whatever the suite turned out to contain, and does not when Playwright
+        // never got that far. `--list` never reaches here at all - onBegin
+        // disables the reporter outright.
+        if (!this.runStarted) {
+            if (result.status !== "passed")
+                return;
+            this.startRun();
+        }
         const deadline = Date.now() + DRAIN_BUDGET_MS;
         const runId = await withDeadline(this.runIdPromise, deadline);
         if (runId === TIMED_OUT) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squally-reporter",
-  "version": "0.5.0",
+  "version": "0.5.2",
   "description": "Playwright reporter that sends test results to your Squally dashboard.",
   "author": "Squally <squally.dev@gmail.com>",
   "keywords": [
@@ -24,7 +24,8 @@
     "prepare": "npm run build",
     "test:unit": "node --test \"test/unit/**/*.test.js\"",
     "test:exit-code": "node test/exit-code/run.mjs",
-    "test": "npm run build && npm run test:unit && npm run test:exit-code"
+    "test:empty-runs": "node test/empty-runs/run.mjs",
+    "test": "npm run build && npm run test:unit && npm run test:exit-code && npm run test:empty-runs"
   },
   "peerDependencies": {
     "@playwright/test": ">=1.30.0"