squally-reporter 0.2.1 → 0.3.0

package/README.md

@@ -50,6 +50,45 @@ reporter: [
 That's it - run your tests as usual (`npx playwright test`) and results will
 be sent to Squally after the run finishes.
 
+## Screenshot/trace upload
+
+If your `playwright.config.ts` captures screenshots or traces, the reporter
+uploads them to Squally so they show up next to the failing test in the
+dashboard - no separate setup beyond the two env vars above.
+
+**Only from attempts worth looking at:** screenshots/traces are uploaded
+only for attempts that **failed**, or that **needed a retry** (even if that
+retry then passed) - never for an attempt that passed on its first try,
+regardless of how `screenshot`/`trace` are configured. Nobody looks at a
+screenshot of a test that just passed, and uploading one anyway would only
+eat into your organization's storage for no benefit.
+
+**You need to actually enable capture in `playwright.config.ts`** - without
+it, Playwright doesn't produce any attachments, so there's nothing for the
+reporter to find or upload (no error either way, it's a silent no-op):
+
+```ts
+export default defineConfig({
+  use: {
+    screenshot: "only-on-failure",
+    trace: "retain-on-failure",
+  },
+  // ...
+});
+```
+
+**Limits:**
+- 50 MB per individual file. Larger files are skipped (logged as a
+  `console.warn`, the test result itself is still reported as usual).
+- A total storage cap per organization, depending on your Squally plan.
+  Once reached, further uploads are skipped the same way until you either
+  free up space or upgrade.
+
+Upload is best-effort and fully non-blocking, same principle as the rest of
+this reporter - a failed or skipped upload never fails your test run or
+changes its exit code, it only logs a `console.warn` explaining what
+happened.
+
 ## Tested revision (optional)
 
 If your test-automation code lives in the same repo as the application it

package/dist/index.d.ts

@@ -6,11 +6,15 @@ interface SquallyReporterOptions {
 export default class SquallyReporter implements Reporter {
     private apiUrl;
     private apiKey;
+    private enabled;
     private startedAt;
     private results;
+    private artifacts;
     constructor(options?: SquallyReporterOptions);
     onBegin(_config: FullConfig, _suite: Suite): void;
     onTestEnd(test: TestCase, result: TestResult): void;
     onEnd(result: FullResult): Promise<void>;
+    private uploadArtifacts;
+    private presignAndUpload;
 }
 export {};

package/dist/index.js

@@ -4,6 +4,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const node_child_process_1 = require("node:child_process");
+const node_fs_1 = require("node:fs");
+const promises_1 = require("node:fs/promises");
 const node_path_1 = __importDefault(require("node:path"));
 const undici_1 = require("undici");
 // Maps Playwright's more granular statuses down to the three
@@ -143,6 +145,29 @@ function detectEventType() {
 function toPosixPath(p) {
     return p.split(node_path_1.default.sep).join("/");
 }
+// Playwright own assertion errors (expect(...).toHaveText(...) etc.) are
+// colorized with ANSI escape codes in error.message/error.stack - useful in
+// a terminal, but rendered as visible garbage once stored as plain text and
+// shown in Squally UI. Test code that colors its own console output (e.g.
+// via chalk/picocolors) can do the same to stdout/stderr. Stripped here,
+// before anything is ever sent - not server/UI-side - so the raw escape
+// codes never land in the database in the first place, and every later
+// consumer (the existing keyword-based error categorization, the planned
+// AI analysis, a fingerprint cache over normalized error signatures)
+// does not have to strip them again itself.
+//
+// Matches the CSI form ANSI color/style codes actually take: an ESC
+// character (code point 27), then an opening bracket, then any run of
+// digits/semicolons, then one letter (e.g. ESC, bracket, 3, 1, m for red).
+// No new dependency (e.g. strip-ansi) for a one-line regex, same reasoning
+// as toPosixPath/the upload worker pool above. Built via String.fromCharCode
+// rather than a hex escape directly in the source: this file has to stay
+// plain, readable ASCII - a raw ESC byte pasted into source code is
+// invisible in most editors/diffs and easy to mistake for empty content.
+function stripAnsi(value) {
+    const ansiPattern = new RegExp(String.fromCharCode(27) + String.fromCharCode(92) + "[[0-9;]*[a-zA-Z]", "g");
+    return value.replace(ansiPattern, "");
+}
 // Playwright captures stdout/stderr as two separate arrays (each internally
 // chronological, but not merged with each other), so a true byte-accurate
 // interleaving between the two streams isn't available here - stdout lines
@@ -152,43 +177,164 @@ function toPosixPath(p) {
 function formatConsoleOutput(result) {
     const lines = [];
     for (const chunk of result.stdout) {
-        lines.push(`[log] ${chunk.toString().replace(/\s+$/, "")}`);
+        lines.push(`[log] ${stripAnsi(chunk.toString()).replace(/\s+$/, "")}`);
     }
     for (const chunk of result.stderr) {
-        lines.push(`[error] ${chunk.toString().replace(/\s+$/, "")}`);
+        lines.push(`[error] ${stripAnsi(chunk.toString()).replace(/\s+$/, "")}`);
     }
     return lines.length > 0 ? lines.join("\n") : null;
 }
+// Presigned upload URLs are valid for 15 minutes (server-side, see
+// squally-app's lib/r2.ts). Uploading fully sequentially could get close to
+// that window for a suite with a very large number of failing/retried
+// tests, but firing everything at once isn't right either - CI runners
+// often have modest outbound bandwidth/file-descriptor limits, and a
+// thundering herd of large trace uploads competes with the test run itself
+// for those resources. 5 concurrent uploads is a pragmatic middle ground:
+// fast enough that even a few hundred artifacts finish in well under 15
+// minutes, bounded enough not to overwhelm a typical CI runner.
+const UPLOAD_CONCURRENCY = 5;
+// No new dependency for this - a fixed-size worker pool that keeps pulling
+// the next item until the list is exhausted is ~10 lines, not worth pulling
+// in e.g. p-limit for.
+async function runWithConcurrencyLimit(items, limit, worker) {
+    let nextIndex = 0;
+    async function runNext() {
+        const currentIndex = nextIndex++;
+        if (currentIndex >= items.length)
+            return;
+        await worker(items[currentIndex]);
+        return runNext();
+    }
+    const workerCount = Math.min(limit, items.length);
+    await Promise.all(Array.from({ length: workerCount }, () => runNext()));
+}
+// Only screenshots/traces from attempts worth looking at get uploaded -
+// deliberately independent of how the customer configured
+// screenshot/trace in playwright.config.ts. A "screenshot: 'on'" config
+// would otherwise attach an image to every single passing test too: nobody
+// opens a screenshot of a test that just passed, and every one of those
+// uploads eats into the organization's artifact cap (Free tier: 1 GB) for
+// zero benefit. "failed" already covers Playwright's timedOut/interrupted
+// (mapStatus above collapses those down) - retryCount > 0 additionally
+// covers a passed-on-retry attempt, since seeing what an earlier, failing
+// attempt looked like is still useful even if the test eventually passed.
+function attemptQualifiesForArtifacts(mappedStatus, retryCount) {
+    return mappedStatus === "failed" || retryCount > 0;
+}
+// A control character as separator, not an empty string or a slash -
+// testName/filePath can contain arbitrary characters, and this key only
+// needs to be unique, not human-readable (it is purely an internal lookup
+// back to the local CollectedArtifact once the server round-trips these
+// identity fields).
+function artifactKey(testName, filePath, retryCount, type) {
+    return [testName, filePath, String(retryCount), type].join(String.fromCharCode(1));
+}
+function skipReasonMessage(reason) {
+    if (reason === "file_too_large")
+        return "file_too_large (limit 50 MB)";
+    if (reason === "cap_exceeded")
+        return "cap_exceeded (storage limit reached, upgrade your plan)";
+    if (reason === "not_found")
+        return "not_found (test result not found on server)";
+    return reason;
+}
+function confirmReasonMessage(reason) {
+    if (reason === "not_uploaded")
+        return "not_uploaded (upload did not reach storage)";
+    if (reason === "not_found")
+        return "not_found (test result not found on server)";
+    return reason;
+}
+function artifactLabel(type) {
+    return type === "screenshot" ? "Screenshot" : "Trace";
+}
 class SquallyReporter {
     constructor(options = {}) {
+        this.apiUrl = "";
+        this.apiKey = "";
+        // Missing/invalid config must never take down the caller's whole test
+        // suite over just a reporting integration - see the constructor below.
+        // Every onXxx hook checks this first and no-ops if it's false.
+        this.enabled = false;
         this.startedAt = new Date();
         this.results = [];
+        this.artifacts = [];
         const apiUrl = options.apiUrl ?? process.env.SQUALLY_API_URL;
         const apiKey = options.apiKey ?? process.env.SQUALLY_API_KEY;
         if (!apiUrl || !apiKey) {
-            throw new Error("[squally-reporter] Missing config: set SQUALLY_API_URL and " +
-                "SQUALLY_API_KEY as env vars, or pass apiUrl/apiKey in playwright.config.ts");
+            // Deliberately NOT thrown - a missing/wrong Squally API key must
+            // only disable reporting to Squally, not fail the caller's entire
+            // CI pipeline before a single test has even run. Warn loudly
+            // instead and continue as a no-op reporter.
+            console.warn("[squally-reporter] no API key configured, skipping report - set " +
+                "SQUALLY_API_URL/SQUALLY_API_KEY as env vars, or pass apiUrl/apiKey " +
+                "in playwright.config.ts");
+            return;
         }
         this.apiUrl = apiUrl.replace(/\/$/, ""); // strip trailing slash if present
         this.apiKey = apiKey;
+        this.enabled = true;
     }
     onBegin(_config, _suite) {
+        if (!this.enabled)
+            return;
         this.startedAt = new Date();
     }
     onTestEnd(test, result) {
+        if (!this.enabled)
+            return;
         const error = result.errors[0];
+        const testName = test.titlePath().slice(1).join(" > ");
+        const filePath = toPosixPath(node_path_1.default.relative(process.cwd(), test.location.file));
+        const status = mapStatus(result.status);
         this.results.push({
-            testName: test.titlePath().slice(1).join(" > "),
-            filePath: toPosixPath(node_path_1.default.relative(process.cwd(), test.location.file)),
-            status: mapStatus(result.status),
+            testName,
+            filePath,
+            status,
             durationMs: result.duration,
             retryCount: result.retry,
-            errorMessage: error?.message ?? null,
-            errorStack: error?.stack ?? null,
+            errorMessage: error?.message ? stripAnsi(error.message) : null,
+            errorStack: error?.stack ? stripAnsi(error.stack) : null,
             consoleOutput: formatConsoleOutput(result),
         });
+        if (!attemptQualifiesForArtifacts(status, result.retry))
+            return;
+        for (const attachment of result.attachments) {
+            if (attachment.name !== "screenshot" && attachment.name !== "trace")
+                continue;
+            // Attachments can be in-memory (body, no path) when added manually via
+            // testInfo.attach() - only file-backed ones (screenshot/trace always
+            // are, when Playwright itself captures them) can be uploaded here.
+            if (!attachment.path)
+                continue;
+            // Synchronous, not fs/promises - onTestEnd's return value isn't
+            // awaited by Playwright (Reporter['onTestEnd'] is typed as void, not
+            // Promise<void>), so an async stat call here would race with
+            // onEnd() reading this.artifacts before it resolved.
+            let sizeBytes;
+            try {
+                sizeBytes = (0, node_fs_1.statSync)(attachment.path).size;
+            }
+            catch {
+                // File already gone (e.g. cleaned up by a parallel worker) -
+                // nothing to upload, silently skip rather than warn for what's an
+                // expected, harmless race in some CI setups.
+                continue;
+            }
+            this.artifacts.push({
+                testName,
+                filePath,
+                retryCount: result.retry,
+                type: attachment.name,
+                localPath: attachment.path,
+                sizeBytes,
+            });
+        }
     }
     async onEnd(result) {
+        if (!this.enabled)
+            return;
         // A run with 0 test results is meaningless for Squally's flakiness
         // calculation and would just be dashboard clutter - regardless of why
         // there were none (npx playwright test --list, an empty testDir, a
@@ -223,26 +369,166 @@ class SquallyReporter {
         // copy when passing our Agent as the dispatcher.
         const dispatcher = new undici_1.Agent({ keepAliveTimeout: 1, keepAliveMaxTimeout: 1 });
         try {
-            const response = await (0, undici_1.fetch)(`${this.apiUrl}/api/ingest`, {
+            let runId = null;
+            try {
+                const response = await (0, undici_1.fetch)(`${this.apiUrl}/api/ingest`, {
+                    method: "POST",
+                    headers: {
+                        "Content-Type": "application/json",
+                        Authorization: `Bearer ${this.apiKey}`,
+                    },
+                    body: JSON.stringify(payload),
+                    dispatcher,
+                });
+                if (!response.ok) {
+                    // Never throw here - a failing reporter upload should not fail the
+                    // whole CI run just because the dashboard was unreachable.
+                    console.error(`[squally-reporter] Ingest failed: ${response.status} ${response.statusText}`);
+                }
+                else {
+                    const body = (await response.json());
+                    runId = body.runId ?? null;
+                }
+            }
+            catch (err) {
+                console.error("[squally-reporter] Failed to reach Squally API:", err);
+            }
+            // Artifact upload is a separate, best-effort step layered on top of
+            // ingest, never a reason to fail it: it needs a runId (presign looks
+            // TestResults up by runId + identity fields, which only exist once
+            // ingest has actually inserted them), and there's nothing to do if
+            // this run had no failing/retried attempts with a screenshot/trace
+            // attached (attemptQualifiesForArtifacts already filtered that in
+            // onTestEnd) - no reason to make an empty presign call.
+            if (runId && this.artifacts.length > 0) {
+                await this.uploadArtifacts(runId, dispatcher);
+            }
+        }
+        finally {
+            await dispatcher.close();
+        }
+    }
+    // Top-level safety net - presignAndUpload already handles presign/
+    // upload/confirm failures individually (see Aufgabe 5's requirement that
+    // none of this ever affects the test suite's exit code), but this catch
+    // still exists in case something in there throws in a way that wasn't
+    // anticipated. The whole artifact feature must be additive: worst case,
+    // some screenshots/traces don't make it to Squally, but the customer's
+    // CI run itself is never at risk.
+    async uploadArtifacts(runId, dispatcher) {
+        try {
+            await this.presignAndUpload(runId, dispatcher);
+        }
+        catch (err) {
+            console.warn("[squally-reporter] Artifact upload failed unexpectedly:", err);
+        }
+    }
+    async presignAndUpload(runId, dispatcher) {
+        const presignArtifacts = this.artifacts.map((artifact) => ({
+            testName: artifact.testName,
+            filePath: artifact.filePath,
+            retryCount: artifact.retryCount,
+            type: artifact.type,
+            sizeBytes: artifact.sizeBytes,
+        }));
+        let presignResults;
+        try {
+            const response = await (0, undici_1.fetch)(`${this.apiUrl}/api/artifacts/presign`, {
                 method: "POST",
                 headers: {
                     "Content-Type": "application/json",
                     Authorization: `Bearer ${this.apiKey}`,
                 },
-                body: JSON.stringify(payload),
+                body: JSON.stringify({ runId, artifacts: presignArtifacts }),
                 dispatcher,
             });
             if (!response.ok) {
-                // Never throw here - a failing reporter upload should not fail the
-                // whole CI run just because the dashboard was unreachable.
-                console.error(`[squally-reporter] Ingest failed: ${response.status} ${response.statusText}`);
+                console.warn(`[squally-reporter] Artifact presign failed: ${response.status} ${response.statusText} - skipping artifact upload for this run`);
+                return;
             }
+            const body = (await response.json());
+            presignResults = body.results;
         }
         catch (err) {
-            console.error("[squally-reporter] Failed to reach Squally API:", err);
+            console.warn("[squally-reporter] Artifact presign request failed:", err);
+            return;
         }
-        finally {
-            await dispatcher.close();
+        // Maps a presign result back to the local file it came from -
+        // testName/filePath/retryCount/type is exactly what was sent, so it's
+        // guaranteed to match one of this.artifacts (barring a server bug),
+        // but a missing lookup is handled defensively below instead of assumed.
+        const artifactsByKey = new Map(this.artifacts.map((artifact) => [
+            artifactKey(artifact.testName, artifact.filePath, artifact.retryCount, artifact.type),
+            artifact,
+        ]));
+        const uploads = [];
+        for (const item of presignResults) {
+            const local = artifactsByKey.get(artifactKey(item.testName, item.filePath ?? "", item.retryCount, item.type));
+            if (!local)
+                continue;
+            if (item.skip) {
+                console.warn(`[squally-reporter] ${artifactLabel(item.type)} for '${item.testName}' skipped: ${skipReasonMessage(item.reason)}`);
+                continue;
+            }
+            uploads.push({ artifact: local, uploadUrl: item.uploadUrl });
+        }
+        if (uploads.length === 0)
+            return;
+        // See UPLOAD_CONCURRENCY's own comment above for why this is bounded
+        // rather than either fully sequential or fully parallel.
+        await runWithConcurrencyLimit(uploads, UPLOAD_CONCURRENCY, async ({ artifact, uploadUrl }) => {
+            try {
+                const fileBody = await (0, promises_1.readFile)(artifact.localPath);
+                const response = await (0, undici_1.fetch)(uploadUrl, {
+                    method: "PUT",
+                    body: fileBody,
+                    dispatcher,
+                });
+                if (!response.ok) {
+                    console.warn(`[squally-reporter] Upload failed for ${artifact.type} of '${artifact.testName}': ${response.status} ${response.statusText}`);
+                }
+            }
+            catch (err) {
+                console.warn(`[squally-reporter] Upload failed for ${artifact.type} of '${artifact.testName}':`, err);
+            }
+        });
+        // Confirm every artifact that was actually presigned (not the ones
+        // presign already skipped - those never got a key, and re-checking
+        // them here would just produce a confusing second "not_uploaded"
+        // warning for something that was never attempted in the first place).
+        // Confirm's own headObject check is the source of truth for whether an
+        // upload really landed - deliberately not tracking PUT success/failure
+        // above to decide what to confirm, since a PUT that reported success
+        // but didn't truly land (or vice versa) should still be caught here.
+        const confirmArtifacts = uploads.map(({ artifact }) => ({
+            testName: artifact.testName,
+            filePath: artifact.filePath,
+            retryCount: artifact.retryCount,
+            type: artifact.type,
+        }));
+        try {
+            const response = await (0, undici_1.fetch)(`${this.apiUrl}/api/artifacts/confirm`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Bearer ${this.apiKey}`,
+                },
+                body: JSON.stringify({ runId, artifacts: confirmArtifacts }),
+                dispatcher,
+            });
+            if (!response.ok) {
+                console.warn(`[squally-reporter] Artifact confirm failed: ${response.status} ${response.statusText}`);
+                return;
+            }
+            const body = (await response.json());
+            for (const item of body.results) {
+                if (item.confirmed)
+                    continue;
+                console.warn(`[squally-reporter] ${artifactLabel(item.type)} for '${item.testName}' not confirmed: ${confirmReasonMessage(item.reason)}`);
+            }
+        }
+        catch (err) {
+            console.warn("[squally-reporter] Artifact confirm request failed:", err);
         }
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squally-reporter",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "description": "Playwright reporter that sends test results to your Squally dashboard.",
   "author": "Squally <squally.dev@gmail.com>",
   "keywords": [