squally-reporter 0.5.2 → 0.6.0

package/dist/index.d.ts

@@ -29,6 +29,18 @@ interface AppendResultEntry {
 type ArtifactType = "screenshot" | "trace";
 declare function truncate(value: string, max: number): string;
 declare function truncateOrNull(value: string | null, max: number): string | null;
+/**
+ * This shard's flush interval: the base cadence stretched by how many shards
+ * share the rate budget with it.
+ *
+ * Only the TIMER is scaled. FLUSH_THRESHOLD_RESULTS is not, and that is
+ * deliberate: it is a latency bound on a fast suite, not a rate control, and
+ * stretching it would delay results without bounding the burst any better -
+ * the burst is bounded by how fast tests finish, which is not ours to set.
+ * The spec's acknowledged worst case (a very fast unit-style suite crossing 25
+ * faster than the interval) is therefore unchanged by this.
+ */
+declare function flushIntervalFor(shardTotal: number | null): number;
 declare function mapStatus(status: TestResult["status"]): AppendResultEntry["status"];
 declare function detectCiRunId(env: NodeJS.ProcessEnv, generateFallback: () => string): string;
 interface ProjectSuiteLike {
@@ -66,6 +78,33 @@ export default class SquallyReporter implements Reporter {
     private runConfig;
     private buffer;
     private artifacts;
+    /**
+     * Every result this shard produced, counted where it is produced (0.6.0).
+     *
+     * SIX PATHS LOSE RESULTS, and this counter exists because five of them are
+     * invisible afterwards and the sixth cannot be counted at all where it
+     * happens:
+     *
+     *   1. a batch dropped after its retries are spent (429, 5xx, network,
+     *      unparseable response) - sendBatch's default branch
+     *   2. a batch dropped as malformed (400), which is not retried
+     *   3. the whole unsent buffer discarded by giveUp() on 402/404/409
+     *   4. every result produced AFTER a give-up - onTestEnd returns before the
+     *      buffer, so there is nothing to count at the point of loss
+     *   5. batches still in flight when the drain budget expires at onEnd; the
+     *      process then completes the run and exits out from under them
+     *   6. onBegin never getting a runId, so nothing was ever sendable
+     *
+     * A drop counter would have to be correct in all six places forever, and
+     * path 4 has no place to put it. Counting production instead needs one
+     * increment in one function, and it makes loss visible even on a path this
+     * reporter does not yet know about - the server compares this against what
+     * it actually stored.
+     *
+     * Per shard, for this shard's runId: each shard reports its own production
+     * against its own row, so no group arithmetic is needed on either side.
+     */
+    private resultsProduced;
     private sendingDisabled;
     private flushTimerHandle;
     private inFlightFlushes;
@@ -116,6 +155,7 @@ export declare const __internal: {
     mapErrors: typeof mapErrors;
     parseRetryAfterMs: typeof parseRetryAfterMs;
     computeBackoffMs: typeof computeBackoffMs;
+    flushIntervalFor: typeof flushIntervalFor;
     attemptQualifiesForArtifacts: typeof attemptQualifiesForArtifacts;
     artifactKey: typeof artifactKey;
 };

package/dist/index.js

@@ -138,6 +138,47 @@ const RULES_REQUEST_TIMEOUT_MS = 5000;
 // API key): 60 s / 2 s = 30 requests/minute, half of 60.
 const FLUSH_THRESHOLD_RESULTS = 25;
 const FLUSH_INTERVAL_MS = 2000;
+// Scaled by the shard count (0.6.0), because the budget the 2 s above was
+// derived from is NOT per reporter - it is per API key, and every shard of a
+// run carries the same key. The firewall counts `header:authorization`, so
+// eight shards each flushing every 2 s produce 8 x 30 = 240 requests/minute
+// against a 60/minute rule. The "half of 60" margin the constant above claims
+// was computed for a single reporter and silently stopped holding the day
+// sharding arrived.
+//
+// interval x shards keeps the AGGREGATE rate at 30/minute for any shard count:
+// S shards x (60 s / 2S s) = 30. The live view does not get slower for it
+// either - it shows the RUN, and a run made of S shards receives an update
+// from one shard or another every 2 s on average regardless of S.
+//
+// CAPPED, and the cap is about crash exposure rather than rate. Everything
+// buffered is lost if the runner is killed (CI cancellation, OOM), so the
+// interval is also the size of the window at risk. 30 s is the same figure
+// RATE_LIMIT_BACKOFF_CAP_MS uses for "the longest we ever wait", and it keeps
+// the aggregate inside the budget up to 30 shards (S x 60/30 <= 60), which
+// covers the 4-to-16 range the app's own shard documentation describes with
+// room to spare.
+//
+// Above 30 shards the aggregate creeps over the rule again. That is accepted:
+// it is outside the documented range, the retry path already handles a 429,
+// and since 0.6.0 any batch actually lost to it is visible in the
+// resultsProduced diff rather than silent.
+const FLUSH_INTERVAL_CAP_MS = 30000;
+/**
+ * This shard's flush interval: the base cadence stretched by how many shards
+ * share the rate budget with it.
+ *
+ * Only the TIMER is scaled. FLUSH_THRESHOLD_RESULTS is not, and that is
+ * deliberate: it is a latency bound on a fast suite, not a rate control, and
+ * stretching it would delay results without bounding the burst any better -
+ * the burst is bounded by how fast tests finish, which is not ours to set.
+ * The spec's acknowledged worst case (a very fast unit-style suite crossing 25
+ * faster than the interval) is therefore unchanged by this.
+ */
+function flushIntervalFor(shardTotal) {
+    const shards = Math.max(1, shardTotal ?? 1);
+    return Math.min(FLUSH_INTERVAL_CAP_MS, FLUSH_INTERVAL_MS * shards);
+}
 // --- Retry policy (spec section 3.2) ----------------------------------------
 //
 // Two independent policies, both applied by requestJson() below via separate
@@ -772,6 +813,33 @@ class SquallyReporter {
         this.projectName = null;
         this.buffer = [];
         this.artifacts = [];
+        /**
+         * Every result this shard produced, counted where it is produced (0.6.0).
+         *
+         * SIX PATHS LOSE RESULTS, and this counter exists because five of them are
+         * invisible afterwards and the sixth cannot be counted at all where it
+         * happens:
+         *
+         *   1. a batch dropped after its retries are spent (429, 5xx, network,
+         *      unparseable response) - sendBatch's default branch
+         *   2. a batch dropped as malformed (400), which is not retried
+         *   3. the whole unsent buffer discarded by giveUp() on 402/404/409
+         *   4. every result produced AFTER a give-up - onTestEnd returns before the
+         *      buffer, so there is nothing to count at the point of loss
+         *   5. batches still in flight when the drain budget expires at onEnd; the
+         *      process then completes the run and exits out from under them
+         *   6. onBegin never getting a runId, so nothing was ever sendable
+         *
+         * A drop counter would have to be correct in all six places forever, and
+         * path 4 has no place to put it. Counting production instead needs one
+         * increment in one function, and it makes loss visible even on a path this
+         * reporter does not yet know about - the server compares this against what
+         * it actually stored.
+         *
+         * Per shard, for this shard's runId: each shard reports its own production
+         * against its own row, so no group arithmetic is needed on either side.
+         */
+        this.resultsProduced = 0;
         // True once nothing more will ever be sent for this run - a permanent
         // give-up (402/404/409 on append, or onBegin never getting a runId at
         // all). onTestEnd stops collecting further results/artifacts once this is
@@ -945,7 +1013,21 @@ class SquallyReporter {
         };
     }
     onTestEnd(test, result) {
-        if (!this.enabled || this.sendingDisabled)
+        // Two guards, deliberately split by the counter between them.
+        //
+        // `enabled` is false when there is nothing to report to at all (no URL or
+        // key, or `--list`): no run exists, no /complete will be sent, and there
+        // is nothing for a count to be compared against.
+        //
+        // `sendingDisabled` is different - a run DOES exist server-side, it will
+        // be completed, and every result arriving after the give-up is lost. That
+        // is loss path 4, and counting BEFORE this return is the only place it can
+        // be seen. Moving the increment below the return would make the largest
+        // loss path of the six silently report zero.
+        if (!this.enabled)
+            return;
+        this.resultsProduced += 1;
+        if (this.sendingDisabled)
             return;
         // Belt and braces: onTestBegin has already done this for every test
         // Playwright executes, but a result arriving without its begin would
@@ -1114,7 +1196,11 @@ class SquallyReporter {
         const handle = setTimeout(() => {
             this.triggerFlush();
             this.armFlushTimer();
-        }, FLUSH_INTERVAL_MS);
+            // Read per arming rather than cached at onBegin: shardTotal is known
+            // before the first arming (config.shard.total, set in onBegin), so this
+            // is the same value every time - reading it here keeps the timer honest
+            // if that ever stops being true.
+        }, flushIntervalFor(this.shardTotal));
         handle.unref();
         this.flushTimerHandle = handle;
     }
@@ -1254,6 +1340,9 @@ class SquallyReporter {
             shardIndex: this.shardIndex,
             status,
             finishedAt: new Date().toISOString(),
+            // Read at the last possible moment: every loss path has already happened
+            // by the time /complete is sent, so this number is final.
+            resultsProduced: this.resultsProduced,
         };
         const outcome = await this.requestJson("POST", `${this.apiUrl}/api/ingest/v2/runs/${runId}/complete`, payload);
         if (outcome.kind !== "ok") {
@@ -1515,6 +1604,7 @@ exports.__internal = {
     mapErrors,
     parseRetryAfterMs,
     computeBackoffMs,
+    flushIntervalFor,
     attemptQualifiesForArtifacts,
     artifactKey,
 };

package/package.json

@@ -1,42 +1,42 @@
-{
-  "name": "squally-reporter",
-  "version": "0.5.2",
-  "description": "Playwright reporter that sends test results to your Squally dashboard.",
-  "author": "Squally <squally.dev@gmail.com>",
-  "keywords": [
-    "playwright",
-    "playwright-reporter",
-    "flaky-tests",
-    "test-results",
-    "ci"
-  ],
-  "main": "dist/index.js",
-  "types": "dist/index.d.ts",
-  "files": [
-    "dist"
-  ],
-  "engines": {
-    "node": ">=22.19.0"
-  },
-  "scripts": {
-    "build": "tsc",
-    "prepublishOnly": "npm run build",
-    "prepare": "npm run build",
-    "test:unit": "node --test \"test/unit/**/*.test.js\"",
-    "test:exit-code": "node test/exit-code/run.mjs",
-    "test:empty-runs": "node test/empty-runs/run.mjs",
-    "test": "npm run build && npm run test:unit && npm run test:exit-code && npm run test:empty-runs"
-  },
-  "peerDependencies": {
-    "@playwright/test": ">=1.30.0"
-  },
-  "devDependencies": {
-    "@playwright/test": "^1.48.0",
-    "@types/node": "^20.14.0",
-    "typescript": "^5.6.0"
-  },
-  "license": "MIT",
-  "dependencies": {
-    "undici": "^8.10.0"
-  }
-}
+{
+  "name": "squally-reporter",
+  "version": "0.6.0",
+  "description": "Playwright reporter that sends test results to your Squally dashboard.",
+  "author": "Squally <squally.dev@gmail.com>",
+  "keywords": [
+    "playwright",
+    "playwright-reporter",
+    "flaky-tests",
+    "test-results",
+    "ci"
+  ],
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "engines": {
+    "node": ">=22.19.0"
+  },
+  "scripts": {
+    "build": "tsc",
+    "prepublishOnly": "npm run build",
+    "prepare": "npm run build",
+    "test:unit": "node --test \"test/unit/**/*.test.js\"",
+    "test:exit-code": "node test/exit-code/run.mjs",
+    "test:empty-runs": "node test/empty-runs/run.mjs",
+    "test": "npm run build && npm run test:unit && npm run test:exit-code && npm run test:empty-runs"
+  },
+  "peerDependencies": {
+    "@playwright/test": ">=1.30.0"
+  },
+  "devDependencies": {
+    "@playwright/test": "^1.48.0",
+    "@types/node": "^20.14.0",
+    "typescript": "^5.6.0"
+  },
+  "license": "MIT",
+  "dependencies": {
+    "undici": "^8.10.0"
+  }
+}