testlens-playwright-reporter 0.2.7 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +87 -0
- package/index.d.ts +40 -0
- package/index.js +168 -3
- package/index.ts +190 -3
- package/lib/index.js +147 -3
- package/package.json +1 -1
package/README.md
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# TestLens Playwright Reporter
|
|
2
|
+
|
|
3
|
+
A Playwright reporter for [TestLens](https://testlens.qa-path.com) - real-time test monitoring dashboard.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- 🚀 **Real-time streaming** - Watch test results as they happen in the dashboard
|
|
8
|
+
- 📸 **Artifact support** - Shows screenshots, videos, and traces
|
|
9
|
+
- 🔄 **Retry tracking** - Monitor test retries and identify flaky tests
|
|
10
|
+
- ⚡ **Cross-platform** - Works on Windows, macOS, and Linux
|
|
11
|
+
|
|
12
|
+
## Installation
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
npm install testlens-playwright-reporter
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Configuration
|
|
19
|
+
|
|
20
|
+
### TypeScript (`playwright.config.ts`)
|
|
21
|
+
|
|
22
|
+
```typescript
|
|
23
|
+
import { defineConfig } from '@playwright/test';
|
|
24
|
+
|
|
25
|
+
export default defineConfig({
|
|
26
|
+
use: {
|
|
27
|
+
// Enable these for better debugging and artifact capture
|
|
28
|
+
screenshot: 'on',
|
|
29
|
+
video: 'on',
|
|
30
|
+
trace: 'on',
|
|
31
|
+
},
|
|
32
|
+
reporter: [
|
|
33
|
+
['testlens-playwright-reporter', {
|
|
34
|
+
apiKey: 'your-api-key-here',
|
|
35
|
+
}]
|
|
36
|
+
],
|
|
37
|
+
});
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
### JavaScript (`playwright.config.js`)
|
|
41
|
+
|
|
42
|
+
```javascript
|
|
43
|
+
const { defineConfig } = require('@playwright/test');
|
|
44
|
+
|
|
45
|
+
module.exports = defineConfig({
|
|
46
|
+
use: {
|
|
47
|
+
// Enable these for better debugging and artifact capture
|
|
48
|
+
screenshot: 'on',
|
|
49
|
+
video: 'on',
|
|
50
|
+
trace: 'on',
|
|
51
|
+
},
|
|
52
|
+
reporter: [
|
|
53
|
+
['testlens-playwright-reporter', {
|
|
54
|
+
apiKey: 'your-api-key-here',
|
|
55
|
+
}]
|
|
56
|
+
],
|
|
57
|
+
});
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
> 💡 **Tip:** Keep `screenshot`, `video`, and `trace` set to `'on'` for better debugging experience. TestLens automatically uploads these artifacts for failed tests, making it easier to identify issues.
|
|
61
|
+
|
|
62
|
+
### Configuration Options
|
|
63
|
+
|
|
64
|
+
| Option | Type | Default | Description |
|
|
65
|
+
|--------|------|---------|-------------|
|
|
66
|
+
| `apiKey` | `string` | **Required** | Your TestLens API key |
|
|
67
|
+
|
|
68
|
+
## Artifacts
|
|
69
|
+
|
|
70
|
+
TestLens automatically captures and uploads:
|
|
71
|
+
|
|
72
|
+
| Artifact | Description |
|
|
73
|
+
|----------|-------------|
|
|
74
|
+
| **Screenshots** | Visual snapshots of test failures |
|
|
75
|
+
| **Videos** | Full video recording of test execution |
|
|
76
|
+
| **Traces** | Playwright trace files for step-by-step debugging |
|
|
77
|
+
|
|
78
|
+
These artifacts are viewable directly in the TestLens dashboard for easy debugging.
|
|
79
|
+
|
|
80
|
+
## Requirements
|
|
81
|
+
|
|
82
|
+
- Node.js >= 16.0.0
|
|
83
|
+
- Playwright >= 1.40.0
|
|
84
|
+
|
|
85
|
+
## License
|
|
86
|
+
|
|
87
|
+
MIT License
|
package/index.d.ts
CHANGED
|
@@ -70,6 +70,46 @@ export interface GitInfo {
|
|
|
70
70
|
remoteUrl: string;
|
|
71
71
|
}
|
|
72
72
|
|
|
73
|
+
/** Rich error details from Playwright */
|
|
74
|
+
export interface TestError {
|
|
75
|
+
message: string;
|
|
76
|
+
stack?: string;
|
|
77
|
+
location?: {
|
|
78
|
+
file: string;
|
|
79
|
+
line: number;
|
|
80
|
+
column: number;
|
|
81
|
+
};
|
|
82
|
+
snippet?: string;
|
|
83
|
+
expected?: string;
|
|
84
|
+
actual?: string;
|
|
85
|
+
diff?: string;
|
|
86
|
+
matcherName?: string;
|
|
87
|
+
timeout?: number;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export interface TestData {
|
|
91
|
+
id: string;
|
|
92
|
+
name: string;
|
|
93
|
+
status: string;
|
|
94
|
+
originalStatus?: string;
|
|
95
|
+
duration: number;
|
|
96
|
+
startTime: string;
|
|
97
|
+
endTime: string;
|
|
98
|
+
errorMessages: string[];
|
|
99
|
+
errors?: TestError[];
|
|
100
|
+
retryAttempts: number;
|
|
101
|
+
currentRetry: number;
|
|
102
|
+
annotations: Array<{ type: string; description?: string }>;
|
|
103
|
+
projectName: string;
|
|
104
|
+
workerIndex?: number;
|
|
105
|
+
parallelIndex?: number;
|
|
106
|
+
location?: {
|
|
107
|
+
file: string;
|
|
108
|
+
line: number;
|
|
109
|
+
column: number;
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
|
|
73
113
|
export default class TestLensReporter implements Reporter {
|
|
74
114
|
constructor(options: TestLensReporterOptions);
|
|
75
115
|
|
package/index.js
CHANGED
|
@@ -213,6 +213,7 @@ class TestLensReporter {
|
|
|
213
213
|
startTime: new Date().toISOString(),
|
|
214
214
|
endTime: '',
|
|
215
215
|
errorMessages: [],
|
|
216
|
+
errors: [],
|
|
216
217
|
retryAttempts: test.retries,
|
|
217
218
|
currentRetry: result.retry,
|
|
218
219
|
annotations: test.annotations.map(ann => ({
|
|
@@ -221,7 +222,12 @@ class TestLensReporter {
|
|
|
221
222
|
})),
|
|
222
223
|
projectName: test.parent.project()?.name || 'default',
|
|
223
224
|
workerIndex: result.workerIndex,
|
|
224
|
-
parallelIndex: result.parallelIndex
|
|
225
|
+
parallelIndex: result.parallelIndex,
|
|
226
|
+
location: {
|
|
227
|
+
file: path.relative(process.cwd(), test.location.file),
|
|
228
|
+
line: test.location.line,
|
|
229
|
+
column: test.location.column
|
|
230
|
+
}
|
|
225
231
|
};
|
|
226
232
|
|
|
227
233
|
this.testMap.set(testData.id, testData);
|
|
@@ -244,7 +250,73 @@ class TestLensReporter {
|
|
|
244
250
|
|
|
245
251
|
async onTestEnd(test, result) {
|
|
246
252
|
const testId = this.getTestId(test);
|
|
247
|
-
|
|
253
|
+
let testData = this.testMap.get(testId);
|
|
254
|
+
|
|
255
|
+
console.log(`[TestLens] onTestEnd called for test: ${test.title}, status: ${result.status}, testData exists: ${!!testData}`);
|
|
256
|
+
|
|
257
|
+
// For skipped tests, onTestBegin might not be called, so we need to create the test data here
|
|
258
|
+
if (!testData) {
|
|
259
|
+
console.log(`[TestLens] Creating test data for skipped/uncreated test: ${test.title}`);
|
|
260
|
+
// Create spec data if not exists (skipped tests might not have spec data either)
|
|
261
|
+
const specPath = test.location.file;
|
|
262
|
+
const specKey = `${specPath}-${test.parent.title}`;
|
|
263
|
+
|
|
264
|
+
if (!this.specMap.has(specKey)) {
|
|
265
|
+
const specData = {
|
|
266
|
+
filePath: path.relative(process.cwd(), specPath),
|
|
267
|
+
testSuiteName: test.parent.title,
|
|
268
|
+
tags: this.extractTags(test),
|
|
269
|
+
startTime: new Date().toISOString(),
|
|
270
|
+
status: 'skipped'
|
|
271
|
+
};
|
|
272
|
+
this.specMap.set(specKey, specData);
|
|
273
|
+
|
|
274
|
+
// Send spec start event to API
|
|
275
|
+
await this.sendToApi({
|
|
276
|
+
type: 'specStart',
|
|
277
|
+
runId: this.runId,
|
|
278
|
+
timestamp: new Date().toISOString(),
|
|
279
|
+
spec: specData
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
// Create test data for skipped test
|
|
284
|
+
testData = {
|
|
285
|
+
id: testId,
|
|
286
|
+
name: test.title,
|
|
287
|
+
status: 'skipped',
|
|
288
|
+
originalStatus: 'skipped',
|
|
289
|
+
duration: 0,
|
|
290
|
+
startTime: new Date().toISOString(),
|
|
291
|
+
endTime: new Date().toISOString(),
|
|
292
|
+
errorMessages: [],
|
|
293
|
+
errors: [],
|
|
294
|
+
retryAttempts: test.retries,
|
|
295
|
+
currentRetry: 0,
|
|
296
|
+
annotations: test.annotations.map(ann => ({
|
|
297
|
+
type: ann.type,
|
|
298
|
+
description: ann.description
|
|
299
|
+
})),
|
|
300
|
+
projectName: test.parent.project()?.name || 'default',
|
|
301
|
+
workerIndex: result.workerIndex,
|
|
302
|
+
parallelIndex: result.parallelIndex,
|
|
303
|
+
location: {
|
|
304
|
+
file: path.relative(process.cwd(), test.location.file),
|
|
305
|
+
line: test.location.line,
|
|
306
|
+
column: test.location.column
|
|
307
|
+
}
|
|
308
|
+
};
|
|
309
|
+
|
|
310
|
+
this.testMap.set(testId, testData);
|
|
311
|
+
|
|
312
|
+
// Send test start event first (so the test gets created in DB)
|
|
313
|
+
await this.sendToApi({
|
|
314
|
+
type: 'testStart',
|
|
315
|
+
runId: this.runId,
|
|
316
|
+
timestamp: new Date().toISOString(),
|
|
317
|
+
test: testData
|
|
318
|
+
});
|
|
319
|
+
}
|
|
248
320
|
|
|
249
321
|
if (testData) {
|
|
250
322
|
// Update test data with latest result
|
|
@@ -254,12 +326,103 @@ class TestLensReporter {
|
|
|
254
326
|
testData.endTime = new Date().toISOString();
|
|
255
327
|
testData.errorMessages = result.errors.map(error => error.message || error.toString());
|
|
256
328
|
testData.currentRetry = result.retry;
|
|
329
|
+
|
|
330
|
+
// Capture test location
|
|
331
|
+
testData.location = {
|
|
332
|
+
file: path.relative(process.cwd(), test.location.file),
|
|
333
|
+
line: test.location.line,
|
|
334
|
+
column: test.location.column
|
|
335
|
+
};
|
|
336
|
+
|
|
337
|
+
// Capture rich error details like Playwright's HTML report
|
|
338
|
+
testData.errors = result.errors.map(error => {
|
|
339
|
+
const testError = {
|
|
340
|
+
message: error.message || error.toString()
|
|
341
|
+
};
|
|
342
|
+
|
|
343
|
+
// Capture stack trace
|
|
344
|
+
if (error.stack) {
|
|
345
|
+
testError.stack = error.stack;
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
// Capture error location
|
|
349
|
+
if (error.location) {
|
|
350
|
+
testError.location = {
|
|
351
|
+
file: path.relative(process.cwd(), error.location.file),
|
|
352
|
+
line: error.location.line,
|
|
353
|
+
column: error.location.column
|
|
354
|
+
};
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
// Capture code snippet around error - from Playwright error object
|
|
358
|
+
if (error.snippet) {
|
|
359
|
+
testError.snippet = error.snippet;
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
// Capture expected/actual values for assertion failures
|
|
363
|
+
// Playwright stores these as specially formatted strings in the message
|
|
364
|
+
const message = error.message || '';
|
|
365
|
+
|
|
366
|
+
// Try to parse expected pattern from toHaveURL and similar assertions
|
|
367
|
+
const expectedPatternMatch = message.match(/Expected pattern:\s*(.+?)(?:\n|$)/);
|
|
368
|
+
if (expectedPatternMatch) {
|
|
369
|
+
testError.expected = expectedPatternMatch[1].trim();
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
// Also try "Expected string:" format
|
|
373
|
+
if (!testError.expected) {
|
|
374
|
+
const expectedStringMatch = message.match(/Expected string:\s*["']?(.+?)["']?(?:\n|$)/);
|
|
375
|
+
if (expectedStringMatch) {
|
|
376
|
+
testError.expected = expectedStringMatch[1].trim();
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
// Try to parse received/actual value
|
|
381
|
+
const receivedMatch = message.match(/Received (?:string|value):\s*["']?(.+?)["']?(?:\n|$)/);
|
|
382
|
+
if (receivedMatch) {
|
|
383
|
+
testError.actual = receivedMatch[1].trim();
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
// Parse call log entries for debugging info (timeouts, retries, etc.)
|
|
387
|
+
const callLogMatch = message.match(/Call log:([\s\S]*?)(?=\n\n|\n\s*\d+\s*\||$)/);
|
|
388
|
+
if (callLogMatch) {
|
|
389
|
+
// Store call log separately for display
|
|
390
|
+
const callLog = callLogMatch[1].trim();
|
|
391
|
+
if (callLog) {
|
|
392
|
+
testError.diff = callLog; // Reuse diff field for call log
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
// Parse timeout information - multiple formats
|
|
397
|
+
const timeoutMatch = message.match(/(?:with timeout|Timeout:?)\s*(\d+)ms/i);
|
|
398
|
+
if (timeoutMatch) {
|
|
399
|
+
testError.timeout = parseInt(timeoutMatch[1], 10);
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
// Parse matcher name (e.g., toHaveURL, toBeVisible)
|
|
403
|
+
const matcherMatch = message.match(/expect\([^)]+\)\.(\w+)/);
|
|
404
|
+
if (matcherMatch) {
|
|
405
|
+
testError.matcherName = matcherMatch[1];
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
// Extract code snippet from message if not already captured
|
|
409
|
+
// Look for lines like " 9 | await page.click..." or "> 11 | await expect..."
|
|
410
|
+
if (!testError.snippet) {
|
|
411
|
+
const codeSnippetMatch = message.match(/((?:\s*>?\s*\d+\s*\|.*\n?)+)/);
|
|
412
|
+
if (codeSnippetMatch) {
|
|
413
|
+
testError.snippet = codeSnippetMatch[1].trim();
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
return testError;
|
|
418
|
+
});
|
|
257
419
|
|
|
258
420
|
// Only send testEnd event after final retry attempt
|
|
259
421
|
// If test passed or this is the last retry, send the event
|
|
260
422
|
const isFinalAttempt = result.status === 'passed' || result.status === 'skipped' || result.retry >= test.retries;
|
|
261
423
|
|
|
262
424
|
if (isFinalAttempt) {
|
|
425
|
+
console.log(`[TestLens] Sending testEnd - testId: ${testData.id}, status: ${testData.status}, originalStatus: ${testData.originalStatus}`);
|
|
263
426
|
// Send test end event to API
|
|
264
427
|
await this.sendToApi({
|
|
265
428
|
type: 'testEnd',
|
|
@@ -645,7 +808,9 @@ class TestLensReporter {
|
|
|
645
808
|
|
|
646
809
|
getTestId(test) {
|
|
647
810
|
const cleanTitle = test.title.replace(/@[\w-]+/g, '').trim();
|
|
648
|
-
|
|
811
|
+
// Normalize path separators to forward slashes for cross-platform consistency
|
|
812
|
+
const normalizedFile = test.location.file.replace(/\\/g, '/');
|
|
813
|
+
return `${normalizedFile}:${test.location.line}:${cleanTitle}`;
|
|
649
814
|
}
|
|
650
815
|
|
|
651
816
|
getFileSize(filePath) {
|
package/index.ts
CHANGED
|
@@ -123,6 +123,22 @@ export interface RunMetadata {
|
|
|
123
123
|
status?: string;
|
|
124
124
|
}
|
|
125
125
|
|
|
126
|
+
export interface TestError {
|
|
127
|
+
message: string;
|
|
128
|
+
stack?: string;
|
|
129
|
+
location?: {
|
|
130
|
+
file: string;
|
|
131
|
+
line: number;
|
|
132
|
+
column: number;
|
|
133
|
+
};
|
|
134
|
+
snippet?: string;
|
|
135
|
+
expected?: string;
|
|
136
|
+
actual?: string;
|
|
137
|
+
diff?: string;
|
|
138
|
+
matcherName?: string;
|
|
139
|
+
timeout?: number;
|
|
140
|
+
}
|
|
141
|
+
|
|
126
142
|
export interface TestData {
|
|
127
143
|
id: string;
|
|
128
144
|
name: string;
|
|
@@ -132,12 +148,18 @@ export interface TestData {
|
|
|
132
148
|
startTime: string;
|
|
133
149
|
endTime: string;
|
|
134
150
|
errorMessages: string[];
|
|
151
|
+
errors?: TestError[]; // Rich error details
|
|
135
152
|
retryAttempts: number;
|
|
136
153
|
currentRetry: number;
|
|
137
154
|
annotations: Array<{ type: string; description?: string }>;
|
|
138
155
|
projectName: string;
|
|
139
156
|
workerIndex?: number;
|
|
140
157
|
parallelIndex?: number;
|
|
158
|
+
location?: {
|
|
159
|
+
file: string;
|
|
160
|
+
line: number;
|
|
161
|
+
column: number;
|
|
162
|
+
};
|
|
141
163
|
}
|
|
142
164
|
|
|
143
165
|
export interface SpecData {
|
|
@@ -350,6 +372,7 @@ export class TestLensReporter implements Reporter {
|
|
|
350
372
|
startTime: new Date().toISOString(),
|
|
351
373
|
endTime: '',
|
|
352
374
|
errorMessages: [],
|
|
375
|
+
errors: [],
|
|
353
376
|
retryAttempts: test.retries,
|
|
354
377
|
currentRetry: result.retry,
|
|
355
378
|
annotations: test.annotations.map((ann: any) => ({
|
|
@@ -358,7 +381,12 @@ export class TestLensReporter implements Reporter {
|
|
|
358
381
|
})),
|
|
359
382
|
projectName: test.parent.project()?.name || 'default',
|
|
360
383
|
workerIndex: result.workerIndex,
|
|
361
|
-
parallelIndex: result.parallelIndex
|
|
384
|
+
parallelIndex: result.parallelIndex,
|
|
385
|
+
location: {
|
|
386
|
+
file: path.relative(process.cwd(), test.location.file),
|
|
387
|
+
line: test.location.line,
|
|
388
|
+
column: test.location.column
|
|
389
|
+
}
|
|
362
390
|
};
|
|
363
391
|
|
|
364
392
|
this.testMap.set(testData.id, testData);
|
|
@@ -381,7 +409,73 @@ export class TestLensReporter implements Reporter {
|
|
|
381
409
|
|
|
382
410
|
async onTestEnd(test: TestCase, result: TestResult): Promise<void> {
|
|
383
411
|
const testId = this.getTestId(test);
|
|
384
|
-
|
|
412
|
+
let testData = this.testMap.get(testId);
|
|
413
|
+
|
|
414
|
+
console.log(`[TestLens] onTestEnd called for test: ${test.title}, status: ${result.status}, testData exists: ${!!testData}`);
|
|
415
|
+
|
|
416
|
+
// For skipped tests, onTestBegin might not be called, so we need to create the test data here
|
|
417
|
+
if (!testData) {
|
|
418
|
+
console.log(`[TestLens] Creating test data for skipped/uncreated test: ${test.title}`);
|
|
419
|
+
// Create spec data if not exists (skipped tests might not have spec data either)
|
|
420
|
+
const specPath = test.location.file;
|
|
421
|
+
const specKey = `${specPath}-${test.parent.title}`;
|
|
422
|
+
|
|
423
|
+
if (!this.specMap.has(specKey)) {
|
|
424
|
+
const specData: SpecData = {
|
|
425
|
+
filePath: path.relative(process.cwd(), specPath),
|
|
426
|
+
testSuiteName: test.parent.title,
|
|
427
|
+
tags: this.extractTags(test),
|
|
428
|
+
startTime: new Date().toISOString(),
|
|
429
|
+
status: 'skipped'
|
|
430
|
+
};
|
|
431
|
+
this.specMap.set(specKey, specData);
|
|
432
|
+
|
|
433
|
+
// Send spec start event to API
|
|
434
|
+
await this.sendToApi({
|
|
435
|
+
type: 'specStart',
|
|
436
|
+
runId: this.runId,
|
|
437
|
+
timestamp: new Date().toISOString(),
|
|
438
|
+
spec: specData
|
|
439
|
+
});
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
// Create test data for skipped test
|
|
443
|
+
testData = {
|
|
444
|
+
id: testId,
|
|
445
|
+
name: test.title,
|
|
446
|
+
status: 'skipped',
|
|
447
|
+
originalStatus: 'skipped',
|
|
448
|
+
duration: 0,
|
|
449
|
+
startTime: new Date().toISOString(),
|
|
450
|
+
endTime: new Date().toISOString(),
|
|
451
|
+
errorMessages: [],
|
|
452
|
+
errors: [],
|
|
453
|
+
retryAttempts: test.retries,
|
|
454
|
+
currentRetry: 0,
|
|
455
|
+
annotations: test.annotations.map((ann: any) => ({
|
|
456
|
+
type: ann.type,
|
|
457
|
+
description: ann.description
|
|
458
|
+
})),
|
|
459
|
+
projectName: test.parent.project()?.name || 'default',
|
|
460
|
+
workerIndex: result.workerIndex,
|
|
461
|
+
parallelIndex: result.parallelIndex,
|
|
462
|
+
location: {
|
|
463
|
+
file: path.relative(process.cwd(), test.location.file),
|
|
464
|
+
line: test.location.line,
|
|
465
|
+
column: test.location.column
|
|
466
|
+
}
|
|
467
|
+
};
|
|
468
|
+
|
|
469
|
+
this.testMap.set(testId, testData);
|
|
470
|
+
|
|
471
|
+
// Send test start event first (so the test gets created in DB)
|
|
472
|
+
await this.sendToApi({
|
|
473
|
+
type: 'testStart',
|
|
474
|
+
runId: this.runId,
|
|
475
|
+
timestamp: new Date().toISOString(),
|
|
476
|
+
test: testData
|
|
477
|
+
});
|
|
478
|
+
}
|
|
385
479
|
|
|
386
480
|
if (testData) {
|
|
387
481
|
// Update test data with latest result
|
|
@@ -391,12 +485,103 @@ export class TestLensReporter implements Reporter {
|
|
|
391
485
|
testData.endTime = new Date().toISOString();
|
|
392
486
|
testData.errorMessages = result.errors.map((error: any) => error.message || error.toString());
|
|
393
487
|
testData.currentRetry = result.retry;
|
|
488
|
+
|
|
489
|
+
// Capture test location
|
|
490
|
+
testData.location = {
|
|
491
|
+
file: path.relative(process.cwd(), test.location.file),
|
|
492
|
+
line: test.location.line,
|
|
493
|
+
column: test.location.column
|
|
494
|
+
};
|
|
495
|
+
|
|
496
|
+
// Capture rich error details like Playwright's HTML report
|
|
497
|
+
testData.errors = result.errors.map((error: any) => {
|
|
498
|
+
const testError: TestError = {
|
|
499
|
+
message: error.message || error.toString()
|
|
500
|
+
};
|
|
501
|
+
|
|
502
|
+
// Capture stack trace
|
|
503
|
+
if (error.stack) {
|
|
504
|
+
testError.stack = error.stack;
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
// Capture error location
|
|
508
|
+
if (error.location) {
|
|
509
|
+
testError.location = {
|
|
510
|
+
file: path.relative(process.cwd(), error.location.file),
|
|
511
|
+
line: error.location.line,
|
|
512
|
+
column: error.location.column
|
|
513
|
+
};
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
// Capture code snippet around error - from Playwright error object
|
|
517
|
+
if (error.snippet) {
|
|
518
|
+
testError.snippet = error.snippet;
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
// Capture expected/actual values for assertion failures
|
|
522
|
+
// Playwright stores these as specially formatted strings in the message
|
|
523
|
+
const message = error.message || '';
|
|
524
|
+
|
|
525
|
+
// Try to parse expected pattern from toHaveURL and similar assertions
|
|
526
|
+
const expectedPatternMatch = message.match(/Expected pattern:\s*(.+?)(?:\n|$)/);
|
|
527
|
+
if (expectedPatternMatch) {
|
|
528
|
+
testError.expected = expectedPatternMatch[1].trim();
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
// Also try "Expected string:" format
|
|
532
|
+
if (!testError.expected) {
|
|
533
|
+
const expectedStringMatch = message.match(/Expected string:\s*["']?(.+?)["']?(?:\n|$)/);
|
|
534
|
+
if (expectedStringMatch) {
|
|
535
|
+
testError.expected = expectedStringMatch[1].trim();
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
// Try to parse received/actual value
|
|
540
|
+
const receivedMatch = message.match(/Received (?:string|value):\s*["']?(.+?)["']?(?:\n|$)/);
|
|
541
|
+
if (receivedMatch) {
|
|
542
|
+
testError.actual = receivedMatch[1].trim();
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
// Parse call log entries for debugging info (timeouts, retries, etc.)
|
|
546
|
+
const callLogMatch = message.match(/Call log:([\s\S]*?)(?=\n\n|\n\s*\d+\s*\||$)/);
|
|
547
|
+
if (callLogMatch) {
|
|
548
|
+
// Store call log separately for display
|
|
549
|
+
const callLog = callLogMatch[1].trim();
|
|
550
|
+
if (callLog) {
|
|
551
|
+
testError.diff = callLog; // Reuse diff field for call log
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
// Parse timeout information - multiple formats
|
|
556
|
+
const timeoutMatch = message.match(/(?:with timeout|Timeout:?)\s*(\d+)ms/i);
|
|
557
|
+
if (timeoutMatch) {
|
|
558
|
+
testError.timeout = parseInt(timeoutMatch[1], 10);
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
// Parse matcher name (e.g., toHaveURL, toBeVisible)
|
|
562
|
+
const matcherMatch = message.match(/expect\([^)]+\)\.(\w+)/);
|
|
563
|
+
if (matcherMatch) {
|
|
564
|
+
testError.matcherName = matcherMatch[1];
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
// Extract code snippet from message if not already captured
|
|
568
|
+
// Look for lines like " 9 | await page.click..." or "> 11 | await expect..."
|
|
569
|
+
if (!testError.snippet) {
|
|
570
|
+
const codeSnippetMatch = message.match(/((?:\s*>?\s*\d+\s*\|.*\n?)+)/);
|
|
571
|
+
if (codeSnippetMatch) {
|
|
572
|
+
testError.snippet = codeSnippetMatch[1].trim();
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
return testError;
|
|
577
|
+
});
|
|
394
578
|
|
|
395
579
|
// Only send testEnd event after final retry attempt
|
|
396
580
|
// If test passed or this is the last retry, send the event
|
|
397
581
|
const isFinalAttempt = result.status === 'passed' || result.status === 'skipped' || result.retry >= test.retries;
|
|
398
582
|
|
|
399
583
|
if (isFinalAttempt) {
|
|
584
|
+
console.log(`[TestLens] Sending testEnd - testId: ${testData.id}, status: ${testData.status}, originalStatus: ${testData.originalStatus}`);
|
|
400
585
|
// Send test end event to API
|
|
401
586
|
await this.sendToApi({
|
|
402
587
|
type: 'testEnd',
|
|
@@ -761,7 +946,9 @@ export class TestLensReporter implements Reporter {
|
|
|
761
946
|
|
|
762
947
|
private getTestId(test: TestCase): string {
|
|
763
948
|
const cleanTitle = test.title.replace(/@[\w-]+/g, '').trim();
|
|
764
|
-
|
|
949
|
+
// Normalize path separators to forward slashes for cross-platform consistency
|
|
950
|
+
const normalizedFile = test.location.file.replace(/\\/g, '/');
|
|
951
|
+
return `${normalizedFile}:${test.location.line}:${cleanTitle}`;
|
|
765
952
|
}
|
|
766
953
|
|
|
767
954
|
|
package/lib/index.js
CHANGED
|
@@ -229,6 +229,7 @@ class TestLensReporter {
|
|
|
229
229
|
startTime: new Date().toISOString(),
|
|
230
230
|
endTime: '',
|
|
231
231
|
errorMessages: [],
|
|
232
|
+
errors: [],
|
|
232
233
|
retryAttempts: test.retries,
|
|
233
234
|
currentRetry: result.retry,
|
|
234
235
|
annotations: test.annotations.map((ann) => ({
|
|
@@ -237,7 +238,12 @@ class TestLensReporter {
|
|
|
237
238
|
})),
|
|
238
239
|
projectName: test.parent.project()?.name || 'default',
|
|
239
240
|
workerIndex: result.workerIndex,
|
|
240
|
-
parallelIndex: result.parallelIndex
|
|
241
|
+
parallelIndex: result.parallelIndex,
|
|
242
|
+
location: {
|
|
243
|
+
file: path.relative(process.cwd(), test.location.file),
|
|
244
|
+
line: test.location.line,
|
|
245
|
+
column: test.location.column
|
|
246
|
+
}
|
|
241
247
|
};
|
|
242
248
|
this.testMap.set(testData.id, testData);
|
|
243
249
|
// Send test start event to API
|
|
@@ -258,7 +264,66 @@ class TestLensReporter {
|
|
|
258
264
|
}
|
|
259
265
|
async onTestEnd(test, result) {
|
|
260
266
|
const testId = this.getTestId(test);
|
|
261
|
-
|
|
267
|
+
let testData = this.testMap.get(testId);
|
|
268
|
+
console.log(`[TestLens] onTestEnd called for test: ${test.title}, status: ${result.status}, testData exists: ${!!testData}`);
|
|
269
|
+
// For skipped tests, onTestBegin might not be called, so we need to create the test data here
|
|
270
|
+
if (!testData) {
|
|
271
|
+
console.log(`[TestLens] Creating test data for skipped/uncreated test: ${test.title}`);
|
|
272
|
+
// Create spec data if not exists (skipped tests might not have spec data either)
|
|
273
|
+
const specPath = test.location.file;
|
|
274
|
+
const specKey = `${specPath}-${test.parent.title}`;
|
|
275
|
+
if (!this.specMap.has(specKey)) {
|
|
276
|
+
const specData = {
|
|
277
|
+
filePath: path.relative(process.cwd(), specPath),
|
|
278
|
+
testSuiteName: test.parent.title,
|
|
279
|
+
tags: this.extractTags(test),
|
|
280
|
+
startTime: new Date().toISOString(),
|
|
281
|
+
status: 'skipped'
|
|
282
|
+
};
|
|
283
|
+
this.specMap.set(specKey, specData);
|
|
284
|
+
// Send spec start event to API
|
|
285
|
+
await this.sendToApi({
|
|
286
|
+
type: 'specStart',
|
|
287
|
+
runId: this.runId,
|
|
288
|
+
timestamp: new Date().toISOString(),
|
|
289
|
+
spec: specData
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
// Create test data for skipped test
|
|
293
|
+
testData = {
|
|
294
|
+
id: testId,
|
|
295
|
+
name: test.title,
|
|
296
|
+
status: 'skipped',
|
|
297
|
+
originalStatus: 'skipped',
|
|
298
|
+
duration: 0,
|
|
299
|
+
startTime: new Date().toISOString(),
|
|
300
|
+
endTime: new Date().toISOString(),
|
|
301
|
+
errorMessages: [],
|
|
302
|
+
errors: [],
|
|
303
|
+
retryAttempts: test.retries,
|
|
304
|
+
currentRetry: 0,
|
|
305
|
+
annotations: test.annotations.map((ann) => ({
|
|
306
|
+
type: ann.type,
|
|
307
|
+
description: ann.description
|
|
308
|
+
})),
|
|
309
|
+
projectName: test.parent.project()?.name || 'default',
|
|
310
|
+
workerIndex: result.workerIndex,
|
|
311
|
+
parallelIndex: result.parallelIndex,
|
|
312
|
+
location: {
|
|
313
|
+
file: path.relative(process.cwd(), test.location.file),
|
|
314
|
+
line: test.location.line,
|
|
315
|
+
column: test.location.column
|
|
316
|
+
}
|
|
317
|
+
};
|
|
318
|
+
this.testMap.set(testId, testData);
|
|
319
|
+
// Send test start event first (so the test gets created in DB)
|
|
320
|
+
await this.sendToApi({
|
|
321
|
+
type: 'testStart',
|
|
322
|
+
runId: this.runId,
|
|
323
|
+
timestamp: new Date().toISOString(),
|
|
324
|
+
test: testData
|
|
325
|
+
});
|
|
326
|
+
}
|
|
262
327
|
if (testData) {
|
|
263
328
|
// Update test data with latest result
|
|
264
329
|
testData.originalStatus = result.status;
|
|
@@ -267,10 +332,87 @@ class TestLensReporter {
|
|
|
267
332
|
testData.endTime = new Date().toISOString();
|
|
268
333
|
testData.errorMessages = result.errors.map((error) => error.message || error.toString());
|
|
269
334
|
testData.currentRetry = result.retry;
|
|
335
|
+
// Capture test location
|
|
336
|
+
testData.location = {
|
|
337
|
+
file: path.relative(process.cwd(), test.location.file),
|
|
338
|
+
line: test.location.line,
|
|
339
|
+
column: test.location.column
|
|
340
|
+
};
|
|
341
|
+
// Capture rich error details like Playwright's HTML report
|
|
342
|
+
testData.errors = result.errors.map((error) => {
|
|
343
|
+
const testError = {
|
|
344
|
+
message: error.message || error.toString()
|
|
345
|
+
};
|
|
346
|
+
// Capture stack trace
|
|
347
|
+
if (error.stack) {
|
|
348
|
+
testError.stack = error.stack;
|
|
349
|
+
}
|
|
350
|
+
// Capture error location
|
|
351
|
+
if (error.location) {
|
|
352
|
+
testError.location = {
|
|
353
|
+
file: path.relative(process.cwd(), error.location.file),
|
|
354
|
+
line: error.location.line,
|
|
355
|
+
column: error.location.column
|
|
356
|
+
};
|
|
357
|
+
}
|
|
358
|
+
// Capture code snippet around error - from Playwright error object
|
|
359
|
+
if (error.snippet) {
|
|
360
|
+
testError.snippet = error.snippet;
|
|
361
|
+
}
|
|
362
|
+
// Capture expected/actual values for assertion failures
|
|
363
|
+
// Playwright stores these as specially formatted strings in the message
|
|
364
|
+
const message = error.message || '';
|
|
365
|
+
// Try to parse expected pattern from toHaveURL and similar assertions
|
|
366
|
+
const expectedPatternMatch = message.match(/Expected pattern:\s*(.+?)(?:\n|$)/);
|
|
367
|
+
if (expectedPatternMatch) {
|
|
368
|
+
testError.expected = expectedPatternMatch[1].trim();
|
|
369
|
+
}
|
|
370
|
+
// Also try "Expected string:" format
|
|
371
|
+
if (!testError.expected) {
|
|
372
|
+
const expectedStringMatch = message.match(/Expected string:\s*["']?(.+?)["']?(?:\n|$)/);
|
|
373
|
+
if (expectedStringMatch) {
|
|
374
|
+
testError.expected = expectedStringMatch[1].trim();
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
// Try to parse received/actual value
|
|
378
|
+
const receivedMatch = message.match(/Received (?:string|value):\s*["']?(.+?)["']?(?:\n|$)/);
|
|
379
|
+
if (receivedMatch) {
|
|
380
|
+
testError.actual = receivedMatch[1].trim();
|
|
381
|
+
}
|
|
382
|
+
// Parse call log entries for debugging info (timeouts, retries, etc.)
|
|
383
|
+
const callLogMatch = message.match(/Call log:([\s\S]*?)(?=\n\n|\n\s*\d+\s*\||$)/);
|
|
384
|
+
if (callLogMatch) {
|
|
385
|
+
// Store call log separately for display
|
|
386
|
+
const callLog = callLogMatch[1].trim();
|
|
387
|
+
if (callLog) {
|
|
388
|
+
testError.diff = callLog; // Reuse diff field for call log
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
// Parse timeout information - multiple formats
|
|
392
|
+
const timeoutMatch = message.match(/(?:with timeout|Timeout:?)\s*(\d+)ms/i);
|
|
393
|
+
if (timeoutMatch) {
|
|
394
|
+
testError.timeout = parseInt(timeoutMatch[1], 10);
|
|
395
|
+
}
|
|
396
|
+
// Parse matcher name (e.g., toHaveURL, toBeVisible)
|
|
397
|
+
const matcherMatch = message.match(/expect\([^)]+\)\.(\w+)/);
|
|
398
|
+
if (matcherMatch) {
|
|
399
|
+
testError.matcherName = matcherMatch[1];
|
|
400
|
+
}
|
|
401
|
+
// Extract code snippet from message if not already captured
|
|
402
|
+
// Look for lines like " 9 | await page.click..." or "> 11 | await expect..."
|
|
403
|
+
if (!testError.snippet) {
|
|
404
|
+
const codeSnippetMatch = message.match(/((?:\s*>?\s*\d+\s*\|.*\n?)+)/);
|
|
405
|
+
if (codeSnippetMatch) {
|
|
406
|
+
testError.snippet = codeSnippetMatch[1].trim();
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
return testError;
|
|
410
|
+
});
|
|
270
411
|
// Only send testEnd event after final retry attempt
|
|
271
412
|
// If test passed or this is the last retry, send the event
|
|
272
413
|
const isFinalAttempt = result.status === 'passed' || result.status === 'skipped' || result.retry >= test.retries;
|
|
273
414
|
if (isFinalAttempt) {
|
|
415
|
+
console.log(`[TestLens] Sending testEnd - testId: ${testData.id}, status: ${testData.status}, originalStatus: ${testData.originalStatus}`);
|
|
274
416
|
// Send test end event to API
|
|
275
417
|
await this.sendToApi({
|
|
276
418
|
type: 'testEnd',
|
|
@@ -599,7 +741,9 @@ class TestLensReporter {
|
|
|
599
741
|
}
|
|
600
742
|
getTestId(test) {
|
|
601
743
|
const cleanTitle = test.title.replace(/@[\w-]+/g, '').trim();
|
|
602
|
-
|
|
744
|
+
// Normalize path separators to forward slashes for cross-platform consistency
|
|
745
|
+
const normalizedFile = test.location.file.replace(/\\/g, '/');
|
|
746
|
+
return `${normalizedFile}:${test.location.line}:${cleanTitle}`;
|
|
603
747
|
}
|
|
604
748
|
async uploadArtifactToS3(filePath, testId, fileName) {
|
|
605
749
|
try {
|
package/package.json
CHANGED