qa-intelligence 1.1.1 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -1
- package/dist/cli/history.js +11 -45
- package/dist/lib/format.js +1 -1
- package/dist/lib/history.d.ts +26 -0
- package/dist/lib/history.js +53 -0
- package/package.json +87 -86
package/README.md
CHANGED
|
@@ -85,7 +85,27 @@ export default defineConfig({
|
|
|
85
85
|
|
|
86
86
|
### 4. Write tests
|
|
87
87
|
|
|
88
|
-
|
|
88
|
+
**Important** — always import `test` and `expect` from the package:
|
|
89
|
+
|
|
90
|
+
```ts
|
|
91
|
+
import { test, expect } from "qa-intelligence/playwright";
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
**Not** directly from Playwright:
|
|
95
|
+
|
|
96
|
+
```ts
|
|
97
|
+
// ❌ No AI artifacts, no PR diff, no flaky detection
|
|
98
|
+
import { test, expect } from "@playwright/test";
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
Using `qa-intelligence/playwright` enables:
|
|
102
|
+
|
|
103
|
+
- AI failure analysis (`meta.json` → `ai.txt`)
|
|
104
|
+
- Artifact generation on failure
|
|
105
|
+
- Flaky detection (retry-aware)
|
|
106
|
+
- CI diff intelligence and PR comments
|
|
107
|
+
|
|
108
|
+
Example test:
|
|
89
109
|
|
|
90
110
|
```ts
|
|
91
111
|
import { test, expect } from "qa-intelligence/playwright";
|
|
@@ -96,6 +116,8 @@ test("user can login", async ({ page }) => {
|
|
|
96
116
|
});
|
|
97
117
|
```
|
|
98
118
|
|
|
119
|
+
> **Using the [framework template](https://github.com/ardithaqi/qa-intelligence-framework)?** Import from local `src/core/baseTest` instead — same hooks, different path for that repo layout.
|
|
120
|
+
|
|
99
121
|
Optional helpers:
|
|
100
122
|
|
|
101
123
|
```ts
|
package/dist/cli/history.js
CHANGED
|
@@ -10,42 +10,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
10
10
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
11
|
const fs_1 = __importDefault(require("fs"));
|
|
12
12
|
const path_1 = __importDefault(require("path"));
|
|
13
|
-
const
|
|
13
|
+
const history_1 = require("../lib/history");
|
|
14
14
|
const HISTORY_PATH = ".cache/failure-history.json";
|
|
15
|
-
const MAX_RUNS = 20;
|
|
16
|
-
function failureKey(f) {
|
|
17
|
-
return (0, failureIdentity_1.failureDiffKey)(f.file, f.failure_type);
|
|
18
|
-
}
|
|
19
|
-
function loadHistory() {
|
|
20
|
-
if (!fs_1.default.existsSync(HISTORY_PATH))
|
|
21
|
-
return { runs: [] };
|
|
22
|
-
try {
|
|
23
|
-
const data = JSON.parse(fs_1.default.readFileSync(HISTORY_PATH, "utf8"));
|
|
24
|
-
return Array.isArray(data.runs) ? { runs: data.runs } : { runs: [] };
|
|
25
|
-
}
|
|
26
|
-
catch {
|
|
27
|
-
return { runs: [] };
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
function enrichFailures(failures, history, currentKeys) {
|
|
31
|
-
const runsIncludingThis = [...history.runs];
|
|
32
|
-
const thisRun = {
|
|
33
|
-
sha: process.env.GITHUB_SHA ?? "",
|
|
34
|
-
timestamp: new Date().toISOString(),
|
|
35
|
-
failureKeys: [...currentKeys],
|
|
36
|
-
};
|
|
37
|
-
runsIncludingThis.push(thisRun);
|
|
38
|
-
return failures.map((f) => {
|
|
39
|
-
const key = failureKey(f);
|
|
40
|
-
const enriched = { ...f };
|
|
41
|
-
const runsWithThis = runsIncludingThis.filter((r) => r.failureKeys.includes(key));
|
|
42
|
-
if (runsWithThis.length > 0) {
|
|
43
|
-
enriched.occurrence_count = runsWithThis.length;
|
|
44
|
-
enriched.first_seen = runsWithThis[0].timestamp;
|
|
45
|
-
}
|
|
46
|
-
return enriched;
|
|
47
|
-
});
|
|
48
|
-
}
|
|
49
15
|
function main() {
|
|
50
16
|
const diffPath = "failure-diff.json";
|
|
51
17
|
if (!fs_1.default.existsSync(diffPath)) {
|
|
@@ -54,17 +20,17 @@ function main() {
|
|
|
54
20
|
}
|
|
55
21
|
const diff = JSON.parse(fs_1.default.readFileSync(diffPath, "utf8"));
|
|
56
22
|
const { newFailures, unchangedFailures, fixedFailures } = diff;
|
|
57
|
-
const currentKeys =
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
const enrichedUnchanged = enrichFailures(unchangedFailures, history, currentKeys);
|
|
64
|
-
const enrichedFixed = enrichFailures(fixedFailures, history, currentKeys);
|
|
65
|
-
const updatedHistory = {
|
|
66
|
-
runs: [...history.runs, { sha: process.env.GITHUB_SHA ?? "", timestamp: new Date().toISOString(), failureKeys: [...currentKeys] }].slice(-MAX_RUNS),
|
|
23
|
+
const currentKeys = (0, history_1.collectCurrentKeys)([...newFailures, ...unchangedFailures]);
|
|
24
|
+
const history = (0, history_1.loadHistory)(HISTORY_PATH);
|
|
25
|
+
const thisRun = {
|
|
26
|
+
sha: process.env.GITHUB_SHA ?? "",
|
|
27
|
+
timestamp: new Date().toISOString(),
|
|
28
|
+
failureKeys: [...currentKeys],
|
|
67
29
|
};
|
|
30
|
+
const enrichedNew = (0, history_1.enrichFailures)(newFailures, history, thisRun);
|
|
31
|
+
const enrichedUnchanged = (0, history_1.enrichFailures)(unchangedFailures, history, thisRun);
|
|
32
|
+
const enrichedFixed = (0, history_1.enrichFailures)(fixedFailures, history, thisRun);
|
|
33
|
+
const updatedHistory = (0, history_1.appendHistoryRun)(history, thisRun);
|
|
68
34
|
fs_1.default.mkdirSync(path_1.default.dirname(HISTORY_PATH), { recursive: true });
|
|
69
35
|
fs_1.default.writeFileSync(HISTORY_PATH, JSON.stringify(updatedHistory, null, 2));
|
|
70
36
|
const result = {
|
package/dist/lib/format.js
CHANGED
|
@@ -46,7 +46,7 @@ function formatDiffComment(diff) {
|
|
|
46
46
|
body += "\n";
|
|
47
47
|
if (unchangedFailures.length > 0) {
|
|
48
48
|
body +=
|
|
49
|
-
"⚠️ **
|
|
49
|
+
"⚠️ **Pre-existing failures** from the base branch are still failing. These are shown for visibility only — **they do not block this PR**. Only **New Issues** block merge.\n\n";
|
|
50
50
|
}
|
|
51
51
|
body += formatSection("New Issues", realNewFailures);
|
|
52
52
|
body += formatSection("Flaky", flaky, false);
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export declare const MAX_RUNS = 20;
|
|
2
|
+
export interface Failure {
|
|
3
|
+
file: string;
|
|
4
|
+
line: number;
|
|
5
|
+
failure_type: string;
|
|
6
|
+
severity: string;
|
|
7
|
+
confidence: number;
|
|
8
|
+
is_flaky_suspected?: boolean;
|
|
9
|
+
}
|
|
10
|
+
export interface EnrichedFailure extends Failure {
|
|
11
|
+
first_seen?: string;
|
|
12
|
+
occurrence_count?: number;
|
|
13
|
+
}
|
|
14
|
+
export interface RunRecord {
|
|
15
|
+
sha: string;
|
|
16
|
+
timestamp: string;
|
|
17
|
+
failureKeys: string[];
|
|
18
|
+
}
|
|
19
|
+
export interface History {
|
|
20
|
+
runs: RunRecord[];
|
|
21
|
+
}
|
|
22
|
+
export declare function failureKey(f: Failure): string;
|
|
23
|
+
export declare function loadHistory(historyPath: string): History;
|
|
24
|
+
export declare function enrichFailures(failures: Failure[], history: History, thisRun: RunRecord): EnrichedFailure[];
|
|
25
|
+
export declare function appendHistoryRun(history: History, run: RunRecord, maxRuns?: number): History;
|
|
26
|
+
export declare function collectCurrentKeys(failures: Failure[]): Set<string>;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.MAX_RUNS = void 0;
|
|
7
|
+
exports.failureKey = failureKey;
|
|
8
|
+
exports.loadHistory = loadHistory;
|
|
9
|
+
exports.enrichFailures = enrichFailures;
|
|
10
|
+
exports.appendHistoryRun = appendHistoryRun;
|
|
11
|
+
exports.collectCurrentKeys = collectCurrentKeys;
|
|
12
|
+
const fs_1 = __importDefault(require("fs"));
|
|
13
|
+
const failureIdentity_1 = require("./failureIdentity");
|
|
14
|
+
exports.MAX_RUNS = 20;
|
|
15
|
+
function failureKey(f) {
|
|
16
|
+
return (0, failureIdentity_1.failureDiffKey)(f.file, f.failure_type);
|
|
17
|
+
}
|
|
18
|
+
function loadHistory(historyPath) {
|
|
19
|
+
if (!fs_1.default.existsSync(historyPath))
|
|
20
|
+
return { runs: [] };
|
|
21
|
+
try {
|
|
22
|
+
const data = JSON.parse(fs_1.default.readFileSync(historyPath, "utf8"));
|
|
23
|
+
return Array.isArray(data.runs) ? { runs: data.runs } : { runs: [] };
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
return { runs: [] };
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
function enrichFailures(failures, history, thisRun) {
|
|
30
|
+
const runsIncludingThis = [...history.runs, thisRun];
|
|
31
|
+
return failures.map((f) => {
|
|
32
|
+
const key = failureKey(f);
|
|
33
|
+
const enriched = { ...f };
|
|
34
|
+
const runsWithThis = runsIncludingThis.filter((r) => r.failureKeys.includes(key));
|
|
35
|
+
if (runsWithThis.length > 0) {
|
|
36
|
+
enriched.occurrence_count = runsWithThis.length;
|
|
37
|
+
enriched.first_seen = runsWithThis[0].timestamp;
|
|
38
|
+
}
|
|
39
|
+
return enriched;
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
function appendHistoryRun(history, run, maxRuns = exports.MAX_RUNS) {
|
|
43
|
+
return {
|
|
44
|
+
runs: [...history.runs, run].slice(-maxRuns),
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
function collectCurrentKeys(failures) {
|
|
48
|
+
const keys = new Set();
|
|
49
|
+
for (const f of failures) {
|
|
50
|
+
keys.add(failureKey(f));
|
|
51
|
+
}
|
|
52
|
+
return keys;
|
|
53
|
+
}
|
package/package.json
CHANGED
|
@@ -1,86 +1,87 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "qa-intelligence",
|
|
3
|
-
"version": "1.1.
|
|
4
|
-
"engines": {
|
|
5
|
-
"node": ">=18"
|
|
6
|
-
},
|
|
7
|
-
"publishConfig": {
|
|
8
|
-
"access": "public"
|
|
9
|
-
},
|
|
10
|
-
"description": "CI intelligence engine for test pipelines. Detects regressions, flaky tests, and failure lifecycle in pull requests.",
|
|
11
|
-
"keywords": [
|
|
12
|
-
"ci",
|
|
13
|
-
"qa",
|
|
14
|
-
"test-automation",
|
|
15
|
-
"playwright",
|
|
16
|
-
"regression-detection",
|
|
17
|
-
"flaky-tests",
|
|
18
|
-
"qa-intelligence"
|
|
19
|
-
],
|
|
20
|
-
"license": "MIT",
|
|
21
|
-
"repository": {
|
|
22
|
-
"type": "git",
|
|
23
|
-
"url": "git+https://github.com/ardithaqi/qa-intelligence.git"
|
|
24
|
-
},
|
|
25
|
-
"type": "commonjs",
|
|
26
|
-
"main": "dist/index.js",
|
|
27
|
-
"exports": {
|
|
28
|
-
"./playwright": {
|
|
29
|
-
"types": "./dist/playwright/baseTest.d.ts",
|
|
30
|
-
"default": "./dist/playwright/baseTest.js"
|
|
31
|
-
},
|
|
32
|
-
"./playwright/globalSetup": {
|
|
33
|
-
"types": "./dist/playwright/globalSetup.d.ts",
|
|
34
|
-
"default": "./dist/playwright/globalSetup.js"
|
|
35
|
-
},
|
|
36
|
-
"./playwright/globalTeardown": {
|
|
37
|
-
"types": "./dist/playwright/globalTeardown.d.ts",
|
|
38
|
-
"default": "./dist/playwright/globalTeardown.js"
|
|
39
|
-
},
|
|
40
|
-
"./playwright/basePage": {
|
|
41
|
-
"types": "./dist/playwright/basePage.d.ts",
|
|
42
|
-
"default": "./dist/playwright/basePage.js"
|
|
43
|
-
},
|
|
44
|
-
"./playwright/steps": {
|
|
45
|
-
"types": "./dist/playwright/steps.d.ts",
|
|
46
|
-
"default": "./dist/playwright/steps.js"
|
|
47
|
-
},
|
|
48
|
-
"./config/env": {
|
|
49
|
-
"types": "./dist/config/env.d.ts",
|
|
50
|
-
"default": "./dist/config/env.js"
|
|
51
|
-
}
|
|
52
|
-
},
|
|
53
|
-
"peerDependencies": {
|
|
54
|
-
"@playwright/test": ">=1.40.0",
|
|
55
|
-
"typescript": ">=5.0.0",
|
|
56
|
-
"@types/node": ">=18.0.0"
|
|
57
|
-
},
|
|
58
|
-
"bin": {
|
|
59
|
-
"qa-intelligence": "dist/cli/index.js",
|
|
60
|
-
"qa-intelligence-diff": "dist/cli/diff.js",
|
|
61
|
-
"qa-intelligence-history": "dist/cli/history.js",
|
|
62
|
-
"qa-intelligence-comment": "dist/cli/postComment.js"
|
|
63
|
-
},
|
|
64
|
-
"scripts": {
|
|
65
|
-
"build": "tsc -p tsconfig.json",
|
|
66
|
-
"
|
|
67
|
-
"
|
|
68
|
-
"dev:
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
"
|
|
73
|
-
"
|
|
74
|
-
"
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
"@
|
|
79
|
-
"
|
|
80
|
-
"
|
|
81
|
-
"
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
1
|
+
{
|
|
2
|
+
"name": "qa-intelligence",
|
|
3
|
+
"version": "1.1.3",
|
|
4
|
+
"engines": {
|
|
5
|
+
"node": ">=18"
|
|
6
|
+
},
|
|
7
|
+
"publishConfig": {
|
|
8
|
+
"access": "public"
|
|
9
|
+
},
|
|
10
|
+
"description": "CI intelligence engine for test pipelines. Detects regressions, flaky tests, and failure lifecycle in pull requests.",
|
|
11
|
+
"keywords": [
|
|
12
|
+
"ci",
|
|
13
|
+
"qa",
|
|
14
|
+
"test-automation",
|
|
15
|
+
"playwright",
|
|
16
|
+
"regression-detection",
|
|
17
|
+
"flaky-tests",
|
|
18
|
+
"qa-intelligence"
|
|
19
|
+
],
|
|
20
|
+
"license": "MIT",
|
|
21
|
+
"repository": {
|
|
22
|
+
"type": "git",
|
|
23
|
+
"url": "git+https://github.com/ardithaqi/qa-intelligence.git"
|
|
24
|
+
},
|
|
25
|
+
"type": "commonjs",
|
|
26
|
+
"main": "dist/index.js",
|
|
27
|
+
"exports": {
|
|
28
|
+
"./playwright": {
|
|
29
|
+
"types": "./dist/playwright/baseTest.d.ts",
|
|
30
|
+
"default": "./dist/playwright/baseTest.js"
|
|
31
|
+
},
|
|
32
|
+
"./playwright/globalSetup": {
|
|
33
|
+
"types": "./dist/playwright/globalSetup.d.ts",
|
|
34
|
+
"default": "./dist/playwright/globalSetup.js"
|
|
35
|
+
},
|
|
36
|
+
"./playwright/globalTeardown": {
|
|
37
|
+
"types": "./dist/playwright/globalTeardown.d.ts",
|
|
38
|
+
"default": "./dist/playwright/globalTeardown.js"
|
|
39
|
+
},
|
|
40
|
+
"./playwright/basePage": {
|
|
41
|
+
"types": "./dist/playwright/basePage.d.ts",
|
|
42
|
+
"default": "./dist/playwright/basePage.js"
|
|
43
|
+
},
|
|
44
|
+
"./playwright/steps": {
|
|
45
|
+
"types": "./dist/playwright/steps.d.ts",
|
|
46
|
+
"default": "./dist/playwright/steps.js"
|
|
47
|
+
},
|
|
48
|
+
"./config/env": {
|
|
49
|
+
"types": "./dist/config/env.d.ts",
|
|
50
|
+
"default": "./dist/config/env.js"
|
|
51
|
+
}
|
|
52
|
+
},
|
|
53
|
+
"peerDependencies": {
|
|
54
|
+
"@playwright/test": ">=1.40.0",
|
|
55
|
+
"typescript": ">=5.0.0",
|
|
56
|
+
"@types/node": ">=18.0.0"
|
|
57
|
+
},
|
|
58
|
+
"bin": {
|
|
59
|
+
"qa-intelligence": "dist/cli/index.js",
|
|
60
|
+
"qa-intelligence-diff": "dist/cli/diff.js",
|
|
61
|
+
"qa-intelligence-history": "dist/cli/history.js",
|
|
62
|
+
"qa-intelligence-comment": "dist/cli/postComment.js"
|
|
63
|
+
},
|
|
64
|
+
"scripts": {
|
|
65
|
+
"build": "tsc -p tsconfig.json",
|
|
66
|
+
"test": "tsx --test src/**/*.test.ts",
|
|
67
|
+
"prepare": "npm run build",
|
|
68
|
+
"dev:diff": "ts-node src/cli/diff.ts --baseline baseline-artifacts --current artifacts --out failure-diff.json",
|
|
69
|
+
"dev:comment": "ts-node src/cli/postComment.ts --diff failure-diff.json --repo owner/repo --pr 1 --token TOKEN"
|
|
70
|
+
},
|
|
71
|
+
"dependencies": {
|
|
72
|
+
"@octokit/rest": "^22.0.1",
|
|
73
|
+
"dotenv": "^17.3.1",
|
|
74
|
+
"openai": "^6.29.0",
|
|
75
|
+
"zod": "^4.3.6"
|
|
76
|
+
},
|
|
77
|
+
"devDependencies": {
|
|
78
|
+
"@playwright/test": "^1.58.2",
|
|
79
|
+
"@types/node": "^22.0.0",
|
|
80
|
+
"ts-node": "^10.9.2",
|
|
81
|
+
"tsx": "^4.21.0",
|
|
82
|
+
"typescript": "^5.9.3"
|
|
83
|
+
},
|
|
84
|
+
"files": [
|
|
85
|
+
"dist"
|
|
86
|
+
]
|
|
87
|
+
}
|