qa-intelligence 1.1.2 → 1.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -125
- package/dist/cli/history.js +11 -45
- package/dist/cli/index.js +39 -0
- package/dist/cli/init.d.ts +10 -0
- package/dist/cli/init.js +99 -0
- package/dist/lib/history.d.ts +26 -0
- package/dist/lib/history.js +53 -0
- package/package.json +89 -86
- package/templates/github/workflows/qa-intelligence.yml +129 -0
- package/templates/playwright/.env.example +8 -0
- package/templates/playwright/package.json +11 -0
- package/templates/playwright/playwright.config.ts +18 -0
- package/templates/playwright/tests/example.spec.ts +6 -0
- package/templates/playwright/tsconfig.json +12 -0
package/README.md
CHANGED
|
@@ -12,135 +12,48 @@ The intelligence engine for Playwright CI pipelines — install it into any proj
|
|
|
12
12
|
|
|
13
13
|
Use it **without the framework template** — install the package into your existing app repo.
|
|
14
14
|
|
|
15
|
-
**Recommended layout:** create a `playwright/` subfolder at the repo root and keep all E2E tooling there, separate from your main app code. Full step-by-step guide (folder layout, `Dockerfile`, CI path changes): **[qa-intelligence-framework README](https://github.com/ardithaqi/qa-intelligence-framework#add-to-an-existing-project-npm-package)**.
|
|
16
15
|
|
|
17
16
|
---
|
|
18
17
|
|
|
19
|
-
##
|
|
18
|
+
## Quick start
|
|
20
19
|
|
|
21
|
-
Run
|
|
20
|
+
Run from your **repo root** (not inside `playwright/`). You do **not** need a `playwright/` folder beforehand — `init` creates it.
|
|
22
21
|
|
|
23
22
|
```bash
|
|
23
|
+
npx qa-intelligence init
|
|
24
24
|
cd playwright
|
|
25
|
-
|
|
25
|
+
cp .env.example .env # set BASE_URL
|
|
26
|
+
npm install # installs qa-intelligence + @playwright/test (see playwright/package.json)
|
|
27
|
+
npx playwright install
|
|
26
28
|
```
|
|
27
29
|
|
|
28
|
-
|
|
30
|
+
`init` scaffolds:
|
|
29
31
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
## Setup
|
|
33
|
-
|
|
34
|
-
All paths below are relative to `playwright/` when using the recommended subfolder layout.
|
|
35
|
-
|
|
36
|
-
### 1. Environment (`.env`)
|
|
37
|
-
|
|
38
|
-
```env
|
|
39
|
-
BASE_URL=https://your-app.example.com
|
|
40
|
-
HEADLESS=true
|
|
41
|
-
PW_WORKERS=2
|
|
42
|
-
PW_RETRIES=1
|
|
43
|
-
```
|
|
44
|
-
|
|
45
|
-
### 2. `tsconfig.json` (TypeScript projects)
|
|
46
|
-
|
|
47
|
-
```json
|
|
48
|
-
{
|
|
49
|
-
"compilerOptions": {
|
|
50
|
-
"target": "ES2022",
|
|
51
|
-
"module": "Node16",
|
|
52
|
-
"moduleResolution": "Node16",
|
|
53
|
-
"strict": true,
|
|
54
|
-
"esModuleInterop": true,
|
|
55
|
-
"types": ["node"],
|
|
56
|
-
"skipLibCheck": true
|
|
57
|
-
},
|
|
58
|
-
"include": ["tests/**/*", "playwright.config.ts"]
|
|
59
|
-
}
|
|
60
|
-
```
|
|
61
|
-
|
|
62
|
-
> `module` and `moduleResolution` must both be `"Node16"` so TypeScript resolves the package `exports` map.
|
|
32
|
+
- `playwright/` — `.env.example`, `package.json`, `tsconfig.json`, `playwright.config.ts`, example test
|
|
33
|
+
- `.github/workflows/qa-intelligence.yml` — PR diff, history, and comment (skip with `--no-ci`)
|
|
63
34
|
|
|
64
|
-
|
|
35
|
+
**Then:** add `OPENAI_API_KEY` to GitHub secrets and set `BASE_URL` in the workflow file.
|
|
65
36
|
|
|
66
|
-
|
|
67
|
-
import { defineConfig } from "@playwright/test";
|
|
68
|
-
import { env } from "qa-intelligence/config/env";
|
|
69
|
-
|
|
70
|
-
export default defineConfig({
|
|
71
|
-
testDir: "./tests",
|
|
72
|
-
retries: env.PW_RETRIES,
|
|
73
|
-
workers: env.PW_WORKERS,
|
|
74
|
-
globalSetup: require.resolve("qa-intelligence/playwright/globalSetup"),
|
|
75
|
-
globalTeardown: require.resolve("qa-intelligence/playwright/globalTeardown"),
|
|
76
|
-
use: {
|
|
77
|
-
baseURL: env.BASE_URL,
|
|
78
|
-
headless: env.HEADLESS,
|
|
79
|
-
trace: "on-first-retry",
|
|
80
|
-
screenshot: "only-on-failure",
|
|
81
|
-
video: "retain-on-failure",
|
|
82
|
-
},
|
|
83
|
-
});
|
|
84
|
-
```
|
|
85
|
-
|
|
86
|
-
### 4. Write tests
|
|
87
|
-
|
|
88
|
-
Always import `test` from the package — **not** directly from Playwright:
|
|
37
|
+
**Tests** — always import from the package, not Playwright directly:
|
|
89
38
|
|
|
90
39
|
```ts
|
|
91
40
|
import { test, expect } from "qa-intelligence/playwright";
|
|
92
|
-
|
|
93
|
-
test("user can login", async ({ page }) => {
|
|
94
|
-
await page.goto("/");
|
|
95
|
-
await expect(page).toHaveTitle(/My App/);
|
|
96
|
-
});
|
|
97
|
-
```
|
|
98
|
-
|
|
99
|
-
Optional helpers:
|
|
100
|
-
|
|
101
|
-
```ts
|
|
102
|
-
import { step } from "qa-intelligence/playwright/steps";
|
|
103
|
-
import { BasePage } from "qa-intelligence/playwright/basePage";
|
|
104
41
|
```
|
|
105
42
|
|
|
106
|
-
###
|
|
43
|
+
### `init` options
|
|
107
44
|
|
|
108
|
-
|
|
109
|
-
|-----------|------------------|
|
|
110
|
-
| `playwright/tests/` | Test hooks, artifact capture |
|
|
111
|
-
| `playwright/.env` | Env validation |
|
|
112
|
-
| `playwright/playwright.config.ts` | `globalSetup`, `globalTeardown`, AI teardown |
|
|
113
|
-
| `playwright/Dockerfile` | — (copy from [framework](https://github.com/ardithaqi/qa-intelligence-framework/blob/master/Dockerfile)) |
|
|
114
|
-
| `.github/workflows/ci.yml` | `qa-intelligence-diff`, `qa-intelligence-history`, `qa-intelligence-comment` |
|
|
115
|
-
| Page objects (optional) | `BasePage`, `step()` |
|
|
45
|
+
By default, `init` skips any file that already exists so it won't overwrite your work.
|
|
116
46
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
Copy [`.github/workflows/ci.yml`](https://github.com/ardithaqi/qa-intelligence-framework/blob/master/.github/workflows/ci.yml) and [`Dockerfile`](https://github.com/ardithaqi/qa-intelligence-framework/blob/master/Dockerfile) from the framework repo.
|
|
122
|
-
|
|
123
|
-
When using the `playwright/` subfolder layout, apply the CI path changes documented in the [framework adoption guide](https://github.com/ardithaqi/qa-intelligence-framework#add-to-an-existing-project-npm-package) (`working-directory: playwright`, artifact paths, etc.).
|
|
124
|
-
|
|
125
|
-
The workflow includes:
|
|
126
|
-
|
|
127
|
-
- Docker test execution
|
|
128
|
-
- Artifact upload on every run
|
|
129
|
-
- Baseline download from `main` on pull requests
|
|
130
|
-
- `qa-intelligence-diff`, `qa-intelligence-history`, `qa-intelligence-comment`
|
|
131
|
-
- PR blocking on new non-flaky failures
|
|
132
|
-
|
|
133
|
-
Then update:
|
|
134
|
-
|
|
135
|
-
- `BASE_URL` in the `docker run` step
|
|
136
|
-
- GitHub secrets (see below)
|
|
47
|
+
| Flag | What it does |
|
|
48
|
+
|------|----------------|
|
|
49
|
+
| `--no-ci` | Only scaffold `playwright/` (config, env, tests). Does **not** create `.github/workflows/qa-intelligence.yml`. Use this if you already have CI or use GitLab/Jenkins. |
|
|
50
|
+
| `--force` | Overwrite existing scaffold files. Use when re-running `init` and you want a fresh copy from the templates. |
|
|
137
51
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
-
|
|
141
|
-
-
|
|
142
|
-
|
|
143
|
-
Set `AI_ANALYSIS=true` in CI when running tests inside Docker.
|
|
52
|
+
```bash
|
|
53
|
+
npx qa-intelligence init # playwright/ + GitHub workflow
|
|
54
|
+
npx qa-intelligence init --no-ci # playwright/ only
|
|
55
|
+
npx qa-intelligence init --force # overwrite files that already exist
|
|
56
|
+
```
|
|
144
57
|
|
|
145
58
|
---
|
|
146
59
|
|
|
@@ -155,20 +68,21 @@ Set `AI_ANALYSIS=true` in CI when running tests inside Docker.
|
|
|
155
68
|
|
|
156
69
|
---
|
|
157
70
|
|
|
158
|
-
## CLI
|
|
71
|
+
## CLI
|
|
159
72
|
|
|
160
73
|
| Command | Purpose |
|
|
161
74
|
|---------|---------|
|
|
75
|
+
| `qa-intelligence init` | Scaffold project files |
|
|
162
76
|
| `qa-intelligence-diff` | Compare baseline vs current failures |
|
|
163
|
-
| `qa-intelligence-history` |
|
|
164
|
-
| `qa-intelligence-comment` | Post/update PR summary
|
|
77
|
+
| `qa-intelligence-history` | Recurrence tracking |
|
|
78
|
+
| `qa-intelligence-comment` | Post/update PR summary |
|
|
165
79
|
|
|
166
80
|
---
|
|
167
81
|
|
|
168
82
|
## Package exports
|
|
169
83
|
|
|
170
|
-
| Import
|
|
171
|
-
|
|
84
|
+
| Import | What you get |
|
|
85
|
+
|--------|--------------|
|
|
172
86
|
| `qa-intelligence/playwright` | `test`, `expect`, `env` |
|
|
173
87
|
| `qa-intelligence/playwright/globalSetup` | Artifact run setup |
|
|
174
88
|
| `qa-intelligence/playwright/globalTeardown` | AI failure analysis |
|
|
@@ -178,19 +92,11 @@ Set `AI_ANALYSIS=true` in CI when running tests inside Docker.
|
|
|
178
92
|
|
|
179
93
|
---
|
|
180
94
|
|
|
181
|
-
##
|
|
182
|
-
|
|
183
|
-
If you prefer a ready-made project with examples, Docker, and CI pre-wired:
|
|
184
|
-
|
|
185
|
-
**[qa-intelligence-framework](https://github.com/ardithaqi/qa-intelligence-framework)** — click "Use this template".
|
|
186
|
-
|
|
187
|
-
The template uses this package under the hood.
|
|
188
|
-
|
|
189
|
-
---
|
|
95
|
+
## Full template (optional)
|
|
190
96
|
|
|
191
|
-
|
|
97
|
+
Prefer a ready-made project with Docker, example tests, and CI pre-wired? Use the **[qa-intelligence-framework](https://github.com/ardithaqi/qa-intelligence-framework)** template — click "Use this template".
|
|
192
98
|
|
|
193
|
-
|
|
99
|
+
For adding to an existing repo with a `playwright/` subfolder, see the [framework adoption guide](https://github.com/ardithaqi/qa-intelligence-framework#add-to-an-existing-project-npm-package) (`Dockerfile`, CI path changes).
|
|
194
100
|
|
|
195
101
|
---
|
|
196
102
|
|
package/dist/cli/history.js
CHANGED
|
@@ -10,42 +10,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
10
10
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
11
|
const fs_1 = __importDefault(require("fs"));
|
|
12
12
|
const path_1 = __importDefault(require("path"));
|
|
13
|
-
const
|
|
13
|
+
const history_1 = require("../lib/history");
|
|
14
14
|
const HISTORY_PATH = ".cache/failure-history.json";
|
|
15
|
-
const MAX_RUNS = 20;
|
|
16
|
-
function failureKey(f) {
|
|
17
|
-
return (0, failureIdentity_1.failureDiffKey)(f.file, f.failure_type);
|
|
18
|
-
}
|
|
19
|
-
function loadHistory() {
|
|
20
|
-
if (!fs_1.default.existsSync(HISTORY_PATH))
|
|
21
|
-
return { runs: [] };
|
|
22
|
-
try {
|
|
23
|
-
const data = JSON.parse(fs_1.default.readFileSync(HISTORY_PATH, "utf8"));
|
|
24
|
-
return Array.isArray(data.runs) ? { runs: data.runs } : { runs: [] };
|
|
25
|
-
}
|
|
26
|
-
catch {
|
|
27
|
-
return { runs: [] };
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
function enrichFailures(failures, history, currentKeys) {
|
|
31
|
-
const runsIncludingThis = [...history.runs];
|
|
32
|
-
const thisRun = {
|
|
33
|
-
sha: process.env.GITHUB_SHA ?? "",
|
|
34
|
-
timestamp: new Date().toISOString(),
|
|
35
|
-
failureKeys: [...currentKeys],
|
|
36
|
-
};
|
|
37
|
-
runsIncludingThis.push(thisRun);
|
|
38
|
-
return failures.map((f) => {
|
|
39
|
-
const key = failureKey(f);
|
|
40
|
-
const enriched = { ...f };
|
|
41
|
-
const runsWithThis = runsIncludingThis.filter((r) => r.failureKeys.includes(key));
|
|
42
|
-
if (runsWithThis.length > 0) {
|
|
43
|
-
enriched.occurrence_count = runsWithThis.length;
|
|
44
|
-
enriched.first_seen = runsWithThis[0].timestamp;
|
|
45
|
-
}
|
|
46
|
-
return enriched;
|
|
47
|
-
});
|
|
48
|
-
}
|
|
49
15
|
function main() {
|
|
50
16
|
const diffPath = "failure-diff.json";
|
|
51
17
|
if (!fs_1.default.existsSync(diffPath)) {
|
|
@@ -54,17 +20,17 @@ function main() {
|
|
|
54
20
|
}
|
|
55
21
|
const diff = JSON.parse(fs_1.default.readFileSync(diffPath, "utf8"));
|
|
56
22
|
const { newFailures, unchangedFailures, fixedFailures } = diff;
|
|
57
|
-
const currentKeys =
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
const enrichedUnchanged = enrichFailures(unchangedFailures, history, currentKeys);
|
|
64
|
-
const enrichedFixed = enrichFailures(fixedFailures, history, currentKeys);
|
|
65
|
-
const updatedHistory = {
|
|
66
|
-
runs: [...history.runs, { sha: process.env.GITHUB_SHA ?? "", timestamp: new Date().toISOString(), failureKeys: [...currentKeys] }].slice(-MAX_RUNS),
|
|
23
|
+
const currentKeys = (0, history_1.collectCurrentKeys)([...newFailures, ...unchangedFailures]);
|
|
24
|
+
const history = (0, history_1.loadHistory)(HISTORY_PATH);
|
|
25
|
+
const thisRun = {
|
|
26
|
+
sha: process.env.GITHUB_SHA ?? "",
|
|
27
|
+
timestamp: new Date().toISOString(),
|
|
28
|
+
failureKeys: [...currentKeys],
|
|
67
29
|
};
|
|
30
|
+
const enrichedNew = (0, history_1.enrichFailures)(newFailures, history, thisRun);
|
|
31
|
+
const enrichedUnchanged = (0, history_1.enrichFailures)(unchangedFailures, history, thisRun);
|
|
32
|
+
const enrichedFixed = (0, history_1.enrichFailures)(fixedFailures, history, thisRun);
|
|
33
|
+
const updatedHistory = (0, history_1.appendHistoryRun)(history, thisRun);
|
|
68
34
|
fs_1.default.mkdirSync(path_1.default.dirname(HISTORY_PATH), { recursive: true });
|
|
69
35
|
fs_1.default.writeFileSync(HISTORY_PATH, JSON.stringify(updatedHistory, null, 2));
|
|
70
36
|
const result = {
|
package/dist/cli/index.js
CHANGED
|
@@ -1,5 +1,38 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
"use strict";
|
|
3
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
4
|
+
if (k2 === undefined) k2 = k;
|
|
5
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
6
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
7
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
8
|
+
}
|
|
9
|
+
Object.defineProperty(o, k2, desc);
|
|
10
|
+
}) : (function(o, m, k, k2) {
|
|
11
|
+
if (k2 === undefined) k2 = k;
|
|
12
|
+
o[k2] = m[k];
|
|
13
|
+
}));
|
|
14
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
15
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
16
|
+
}) : function(o, v) {
|
|
17
|
+
o["default"] = v;
|
|
18
|
+
});
|
|
19
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
20
|
+
var ownKeys = function(o) {
|
|
21
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
22
|
+
var ar = [];
|
|
23
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
24
|
+
return ar;
|
|
25
|
+
};
|
|
26
|
+
return ownKeys(o);
|
|
27
|
+
};
|
|
28
|
+
return function (mod) {
|
|
29
|
+
if (mod && mod.__esModule) return mod;
|
|
30
|
+
var result = {};
|
|
31
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
32
|
+
__setModuleDefault(result, mod);
|
|
33
|
+
return result;
|
|
34
|
+
};
|
|
35
|
+
})();
|
|
3
36
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
37
|
const child_process_1 = require("child_process");
|
|
5
38
|
function getArg(name) {
|
|
@@ -9,6 +42,12 @@ function getArg(name) {
|
|
|
9
42
|
return process.argv[idx + 1];
|
|
10
43
|
}
|
|
11
44
|
async function main() {
|
|
45
|
+
const subcommand = process.argv[2];
|
|
46
|
+
if (subcommand === "init") {
|
|
47
|
+
const { main: runInit } = await Promise.resolve().then(() => __importStar(require("./init")));
|
|
48
|
+
await runInit();
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
12
51
|
const baseline = getArg("baseline") ?? "baseline-artifacts";
|
|
13
52
|
const current = getArg("current") ?? "artifacts";
|
|
14
53
|
const repo = getArg("repo");
|
package/dist/cli/init.js
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
4
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
5
|
+
};
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.runInit = runInit;
|
|
8
|
+
exports.main = main;
|
|
9
|
+
const fs_1 = __importDefault(require("fs"));
|
|
10
|
+
const path_1 = __importDefault(require("path"));
|
|
11
|
+
const PLAYWRIGHT_SCAFFOLD = [
|
|
12
|
+
{ template: "playwright/.env.example", dest: "playwright/.env.example" },
|
|
13
|
+
{ template: "playwright/tsconfig.json", dest: "playwright/tsconfig.json" },
|
|
14
|
+
{
|
|
15
|
+
template: "playwright/playwright.config.ts",
|
|
16
|
+
dest: "playwright/playwright.config.ts",
|
|
17
|
+
},
|
|
18
|
+
{ template: "playwright/package.json", dest: "playwright/package.json" },
|
|
19
|
+
{
|
|
20
|
+
template: "playwright/tests/example.spec.ts",
|
|
21
|
+
dest: "playwright/tests/example.spec.ts",
|
|
22
|
+
},
|
|
23
|
+
];
|
|
24
|
+
const CI_SCAFFOLD = [
|
|
25
|
+
{
|
|
26
|
+
template: "github/workflows/qa-intelligence.yml",
|
|
27
|
+
dest: ".github/workflows/qa-intelligence.yml",
|
|
28
|
+
},
|
|
29
|
+
];
|
|
30
|
+
function hasFlag(name) {
|
|
31
|
+
return process.argv.includes(`--${name}`);
|
|
32
|
+
}
|
|
33
|
+
function templatesDir() {
|
|
34
|
+
return path_1.default.join(__dirname, "..", "..", "templates");
|
|
35
|
+
}
|
|
36
|
+
function copyScaffold(root, entries, force) {
|
|
37
|
+
const written = [];
|
|
38
|
+
const skipped = [];
|
|
39
|
+
const sourceRoot = templatesDir();
|
|
40
|
+
for (const { template, dest } of entries) {
|
|
41
|
+
const source = path_1.default.join(sourceRoot, template);
|
|
42
|
+
const target = path_1.default.join(root, dest);
|
|
43
|
+
if (!fs_1.default.existsSync(source)) {
|
|
44
|
+
throw new Error(`Missing template: ${template}`);
|
|
45
|
+
}
|
|
46
|
+
if (fs_1.default.existsSync(target) && !force) {
|
|
47
|
+
skipped.push(dest);
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
fs_1.default.mkdirSync(path_1.default.dirname(target), { recursive: true });
|
|
51
|
+
fs_1.default.copyFileSync(source, target);
|
|
52
|
+
written.push(dest);
|
|
53
|
+
}
|
|
54
|
+
return { written, skipped };
|
|
55
|
+
}
|
|
56
|
+
function runInit(options) {
|
|
57
|
+
const root = path_1.default.resolve(options?.root ?? process.cwd());
|
|
58
|
+
const force = options?.force ?? false;
|
|
59
|
+
const withCi = options?.withCi ?? true;
|
|
60
|
+
const entries = [...PLAYWRIGHT_SCAFFOLD];
|
|
61
|
+
if (withCi) {
|
|
62
|
+
entries.push(...CI_SCAFFOLD);
|
|
63
|
+
}
|
|
64
|
+
return copyScaffold(root, entries, force);
|
|
65
|
+
}
|
|
66
|
+
async function main() {
|
|
67
|
+
const force = hasFlag("force");
|
|
68
|
+
const noCi = hasFlag("no-ci");
|
|
69
|
+
console.log("Scaffolding qa-intelligence in:", process.cwd());
|
|
70
|
+
const { written, skipped } = runInit({
|
|
71
|
+
force,
|
|
72
|
+
withCi: !noCi,
|
|
73
|
+
});
|
|
74
|
+
for (const file of written) {
|
|
75
|
+
console.log(` created ${file}`);
|
|
76
|
+
}
|
|
77
|
+
for (const file of skipped) {
|
|
78
|
+
console.log(` skipped ${file} (exists — use --force to overwrite)`);
|
|
79
|
+
}
|
|
80
|
+
if (written.length === 0 && skipped.length > 0) {
|
|
81
|
+
console.log("\nNothing new was written.");
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
console.log("\nNext steps:");
|
|
85
|
+
console.log(" 1. cd playwright && cp .env.example .env # set BASE_URL");
|
|
86
|
+
console.log(" 2. cd playwright && npm install");
|
|
87
|
+
console.log(" 3. npx playwright install");
|
|
88
|
+
console.log(" 4. Add OPENAI_API_KEY to GitHub repo secrets (for CI)");
|
|
89
|
+
if (!noCi) {
|
|
90
|
+
console.log(" 5. Update BASE_URL in .github/workflows/qa-intelligence.yml");
|
|
91
|
+
}
|
|
92
|
+
console.log(" 6. Write tests — import from qa-intelligence/playwright");
|
|
93
|
+
}
|
|
94
|
+
if (require.main === module) {
|
|
95
|
+
main().catch((e) => {
|
|
96
|
+
console.error(e);
|
|
97
|
+
process.exit(1);
|
|
98
|
+
});
|
|
99
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export declare const MAX_RUNS = 20;
|
|
2
|
+
export interface Failure {
|
|
3
|
+
file: string;
|
|
4
|
+
line: number;
|
|
5
|
+
failure_type: string;
|
|
6
|
+
severity: string;
|
|
7
|
+
confidence: number;
|
|
8
|
+
is_flaky_suspected?: boolean;
|
|
9
|
+
}
|
|
10
|
+
export interface EnrichedFailure extends Failure {
|
|
11
|
+
first_seen?: string;
|
|
12
|
+
occurrence_count?: number;
|
|
13
|
+
}
|
|
14
|
+
export interface RunRecord {
|
|
15
|
+
sha: string;
|
|
16
|
+
timestamp: string;
|
|
17
|
+
failureKeys: string[];
|
|
18
|
+
}
|
|
19
|
+
export interface History {
|
|
20
|
+
runs: RunRecord[];
|
|
21
|
+
}
|
|
22
|
+
export declare function failureKey(f: Failure): string;
|
|
23
|
+
export declare function loadHistory(historyPath: string): History;
|
|
24
|
+
export declare function enrichFailures(failures: Failure[], history: History, thisRun: RunRecord): EnrichedFailure[];
|
|
25
|
+
export declare function appendHistoryRun(history: History, run: RunRecord, maxRuns?: number): History;
|
|
26
|
+
export declare function collectCurrentKeys(failures: Failure[]): Set<string>;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.MAX_RUNS = void 0;
|
|
7
|
+
exports.failureKey = failureKey;
|
|
8
|
+
exports.loadHistory = loadHistory;
|
|
9
|
+
exports.enrichFailures = enrichFailures;
|
|
10
|
+
exports.appendHistoryRun = appendHistoryRun;
|
|
11
|
+
exports.collectCurrentKeys = collectCurrentKeys;
|
|
12
|
+
const fs_1 = __importDefault(require("fs"));
|
|
13
|
+
const failureIdentity_1 = require("./failureIdentity");
|
|
14
|
+
exports.MAX_RUNS = 20;
|
|
15
|
+
function failureKey(f) {
|
|
16
|
+
return (0, failureIdentity_1.failureDiffKey)(f.file, f.failure_type);
|
|
17
|
+
}
|
|
18
|
+
function loadHistory(historyPath) {
|
|
19
|
+
if (!fs_1.default.existsSync(historyPath))
|
|
20
|
+
return { runs: [] };
|
|
21
|
+
try {
|
|
22
|
+
const data = JSON.parse(fs_1.default.readFileSync(historyPath, "utf8"));
|
|
23
|
+
return Array.isArray(data.runs) ? { runs: data.runs } : { runs: [] };
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
return { runs: [] };
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
function enrichFailures(failures, history, thisRun) {
|
|
30
|
+
const runsIncludingThis = [...history.runs, thisRun];
|
|
31
|
+
return failures.map((f) => {
|
|
32
|
+
const key = failureKey(f);
|
|
33
|
+
const enriched = { ...f };
|
|
34
|
+
const runsWithThis = runsIncludingThis.filter((r) => r.failureKeys.includes(key));
|
|
35
|
+
if (runsWithThis.length > 0) {
|
|
36
|
+
enriched.occurrence_count = runsWithThis.length;
|
|
37
|
+
enriched.first_seen = runsWithThis[0].timestamp;
|
|
38
|
+
}
|
|
39
|
+
return enriched;
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
function appendHistoryRun(history, run, maxRuns = exports.MAX_RUNS) {
|
|
43
|
+
return {
|
|
44
|
+
runs: [...history.runs, run].slice(-maxRuns),
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
function collectCurrentKeys(failures) {
|
|
48
|
+
const keys = new Set();
|
|
49
|
+
for (const f of failures) {
|
|
50
|
+
keys.add(failureKey(f));
|
|
51
|
+
}
|
|
52
|
+
return keys;
|
|
53
|
+
}
|
package/package.json
CHANGED
|
@@ -1,86 +1,89 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "qa-intelligence",
|
|
3
|
-
"version": "1.1.
|
|
4
|
-
"engines": {
|
|
5
|
-
"node": ">=18"
|
|
6
|
-
},
|
|
7
|
-
"publishConfig": {
|
|
8
|
-
"access": "public"
|
|
9
|
-
},
|
|
10
|
-
"description": "CI intelligence engine for test pipelines. Detects regressions, flaky tests, and failure lifecycle in pull requests.",
|
|
11
|
-
"keywords": [
|
|
12
|
-
"ci",
|
|
13
|
-
"qa",
|
|
14
|
-
"test-automation",
|
|
15
|
-
"playwright",
|
|
16
|
-
"regression-detection",
|
|
17
|
-
"flaky-tests",
|
|
18
|
-
"qa-intelligence"
|
|
19
|
-
],
|
|
20
|
-
"license": "MIT",
|
|
21
|
-
"repository": {
|
|
22
|
-
"type": "git",
|
|
23
|
-
"url": "git+https://github.com/ardithaqi/qa-intelligence.git"
|
|
24
|
-
},
|
|
25
|
-
"type": "commonjs",
|
|
26
|
-
"main": "dist/index.js",
|
|
27
|
-
"exports": {
|
|
28
|
-
"./playwright": {
|
|
29
|
-
"types": "./dist/playwright/baseTest.d.ts",
|
|
30
|
-
"default": "./dist/playwright/baseTest.js"
|
|
31
|
-
},
|
|
32
|
-
"./playwright/globalSetup": {
|
|
33
|
-
"types": "./dist/playwright/globalSetup.d.ts",
|
|
34
|
-
"default": "./dist/playwright/globalSetup.js"
|
|
35
|
-
},
|
|
36
|
-
"./playwright/globalTeardown": {
|
|
37
|
-
"types": "./dist/playwright/globalTeardown.d.ts",
|
|
38
|
-
"default": "./dist/playwright/globalTeardown.js"
|
|
39
|
-
},
|
|
40
|
-
"./playwright/basePage": {
|
|
41
|
-
"types": "./dist/playwright/basePage.d.ts",
|
|
42
|
-
"default": "./dist/playwright/basePage.js"
|
|
43
|
-
},
|
|
44
|
-
"./playwright/steps": {
|
|
45
|
-
"types": "./dist/playwright/steps.d.ts",
|
|
46
|
-
"default": "./dist/playwright/steps.js"
|
|
47
|
-
},
|
|
48
|
-
"./config/env": {
|
|
49
|
-
"types": "./dist/config/env.d.ts",
|
|
50
|
-
"default": "./dist/config/env.js"
|
|
51
|
-
}
|
|
52
|
-
},
|
|
53
|
-
"peerDependencies": {
|
|
54
|
-
"@playwright/test": ">=1.40.0",
|
|
55
|
-
"typescript": ">=5.0.0",
|
|
56
|
-
"@types/node": ">=18.0.0"
|
|
57
|
-
},
|
|
58
|
-
"bin": {
|
|
59
|
-
"qa-intelligence": "dist/cli/index.js",
|
|
60
|
-
"qa-intelligence-
|
|
61
|
-
"qa-intelligence-
|
|
62
|
-
"qa-intelligence-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
"
|
|
67
|
-
"
|
|
68
|
-
"
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
"
|
|
74
|
-
"
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
"
|
|
80
|
-
"
|
|
81
|
-
"
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
1
|
+
{
|
|
2
|
+
"name": "qa-intelligence",
|
|
3
|
+
"version": "1.1.4",
|
|
4
|
+
"engines": {
|
|
5
|
+
"node": ">=18"
|
|
6
|
+
},
|
|
7
|
+
"publishConfig": {
|
|
8
|
+
"access": "public"
|
|
9
|
+
},
|
|
10
|
+
"description": "CI intelligence engine for test pipelines. Detects regressions, flaky tests, and failure lifecycle in pull requests.",
|
|
11
|
+
"keywords": [
|
|
12
|
+
"ci",
|
|
13
|
+
"qa",
|
|
14
|
+
"test-automation",
|
|
15
|
+
"playwright",
|
|
16
|
+
"regression-detection",
|
|
17
|
+
"flaky-tests",
|
|
18
|
+
"qa-intelligence"
|
|
19
|
+
],
|
|
20
|
+
"license": "MIT",
|
|
21
|
+
"repository": {
|
|
22
|
+
"type": "git",
|
|
23
|
+
"url": "git+https://github.com/ardithaqi/qa-intelligence.git"
|
|
24
|
+
},
|
|
25
|
+
"type": "commonjs",
|
|
26
|
+
"main": "dist/index.js",
|
|
27
|
+
"exports": {
|
|
28
|
+
"./playwright": {
|
|
29
|
+
"types": "./dist/playwright/baseTest.d.ts",
|
|
30
|
+
"default": "./dist/playwright/baseTest.js"
|
|
31
|
+
},
|
|
32
|
+
"./playwright/globalSetup": {
|
|
33
|
+
"types": "./dist/playwright/globalSetup.d.ts",
|
|
34
|
+
"default": "./dist/playwright/globalSetup.js"
|
|
35
|
+
},
|
|
36
|
+
"./playwright/globalTeardown": {
|
|
37
|
+
"types": "./dist/playwright/globalTeardown.d.ts",
|
|
38
|
+
"default": "./dist/playwright/globalTeardown.js"
|
|
39
|
+
},
|
|
40
|
+
"./playwright/basePage": {
|
|
41
|
+
"types": "./dist/playwright/basePage.d.ts",
|
|
42
|
+
"default": "./dist/playwright/basePage.js"
|
|
43
|
+
},
|
|
44
|
+
"./playwright/steps": {
|
|
45
|
+
"types": "./dist/playwright/steps.d.ts",
|
|
46
|
+
"default": "./dist/playwright/steps.js"
|
|
47
|
+
},
|
|
48
|
+
"./config/env": {
|
|
49
|
+
"types": "./dist/config/env.d.ts",
|
|
50
|
+
"default": "./dist/config/env.js"
|
|
51
|
+
}
|
|
52
|
+
},
|
|
53
|
+
"peerDependencies": {
|
|
54
|
+
"@playwright/test": ">=1.40.0",
|
|
55
|
+
"typescript": ">=5.0.0",
|
|
56
|
+
"@types/node": ">=18.0.0"
|
|
57
|
+
},
|
|
58
|
+
"bin": {
|
|
59
|
+
"qa-intelligence": "dist/cli/index.js",
|
|
60
|
+
"qa-intelligence-init": "dist/cli/init.js",
|
|
61
|
+
"qa-intelligence-diff": "dist/cli/diff.js",
|
|
62
|
+
"qa-intelligence-history": "dist/cli/history.js",
|
|
63
|
+
"qa-intelligence-comment": "dist/cli/postComment.js"
|
|
64
|
+
},
|
|
65
|
+
"scripts": {
|
|
66
|
+
"build": "tsc -p tsconfig.json",
|
|
67
|
+
"test": "tsx --test src/**/*.test.ts",
|
|
68
|
+
"prepare": "npm run build",
|
|
69
|
+
"dev:diff": "ts-node src/cli/diff.ts --baseline baseline-artifacts --current artifacts --out failure-diff.json",
|
|
70
|
+
"dev:comment": "ts-node src/cli/postComment.ts --diff failure-diff.json --repo owner/repo --pr 1 --token TOKEN"
|
|
71
|
+
},
|
|
72
|
+
"dependencies": {
|
|
73
|
+
"@octokit/rest": "^22.0.1",
|
|
74
|
+
"dotenv": "^17.3.1",
|
|
75
|
+
"openai": "^6.29.0",
|
|
76
|
+
"zod": "^4.3.6"
|
|
77
|
+
},
|
|
78
|
+
"devDependencies": {
|
|
79
|
+
"@playwright/test": "^1.58.2",
|
|
80
|
+
"@types/node": "^22.0.0",
|
|
81
|
+
"ts-node": "^10.9.2",
|
|
82
|
+
"tsx": "^4.21.0",
|
|
83
|
+
"typescript": "^5.9.3"
|
|
84
|
+
},
|
|
85
|
+
"files": [
|
|
86
|
+
"dist",
|
|
87
|
+
"templates"
|
|
88
|
+
]
|
|
89
|
+
}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
name: QA Intelligence CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main, master]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main, master]
|
|
8
|
+
|
|
9
|
+
permissions:
|
|
10
|
+
contents: read
|
|
11
|
+
actions: read
|
|
12
|
+
issues: write
|
|
13
|
+
pull-requests: write
|
|
14
|
+
|
|
15
|
+
jobs:
|
|
16
|
+
test:
|
|
17
|
+
runs-on: ubuntu-latest
|
|
18
|
+
defaults:
|
|
19
|
+
run:
|
|
20
|
+
working-directory: playwright
|
|
21
|
+
|
|
22
|
+
steps:
|
|
23
|
+
- uses: actions/checkout@v4
|
|
24
|
+
|
|
25
|
+
- uses: actions/setup-node@v4
|
|
26
|
+
with:
|
|
27
|
+
node-version: 20
|
|
28
|
+
|
|
29
|
+
- name: Install dependencies
|
|
30
|
+
run: npm install
|
|
31
|
+
|
|
32
|
+
- name: Install Playwright browsers
|
|
33
|
+
run: npx playwright install --with-deps chromium
|
|
34
|
+
|
|
35
|
+
- name: Clean artifacts
|
|
36
|
+
if: always()
|
|
37
|
+
run: rm -rf artifacts
|
|
38
|
+
|
|
39
|
+
- name: Run Playwright tests
|
|
40
|
+
env:
|
|
41
|
+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
42
|
+
AI_ANALYSIS: true
|
|
43
|
+
BASE_URL: https://your-app.example.com
|
|
44
|
+
HEADLESS: true
|
|
45
|
+
PW_WORKERS: 2
|
|
46
|
+
PW_RETRIES: 1
|
|
47
|
+
TEST_USERNAME: ${{ secrets.TEST_USERNAME }}
|
|
48
|
+
TEST_PASSWORD: ${{ secrets.TEST_PASSWORD }}
|
|
49
|
+
run: |
|
|
50
|
+
set +e
|
|
51
|
+
npx playwright test
|
|
52
|
+
EXIT=$?
|
|
53
|
+
if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then exit 0; fi
|
|
54
|
+
exit $EXIT
|
|
55
|
+
|
|
56
|
+
- name: Upload Playwright HTML report
|
|
57
|
+
if: always()
|
|
58
|
+
uses: actions/upload-artifact@v4
|
|
59
|
+
with:
|
|
60
|
+
name: playwright-report
|
|
61
|
+
path: playwright/playwright-report/
|
|
62
|
+
|
|
63
|
+
- name: Upload AI failure artifacts
|
|
64
|
+
if: always()
|
|
65
|
+
uses: actions/upload-artifact@v4
|
|
66
|
+
with:
|
|
67
|
+
name: ai-failure-artifacts
|
|
68
|
+
path: playwright/artifacts/
|
|
69
|
+
|
|
70
|
+
- name: Parse AI failure reports
|
|
71
|
+
if: always()
|
|
72
|
+
run: |
|
|
73
|
+
echo "Parsing AI failure reports..."
|
|
74
|
+
if [ -d "artifacts" ]; then
|
|
75
|
+
find artifacts -name "ai.txt" | while read file; do
|
|
76
|
+
echo "-----"
|
|
77
|
+
echo "File: $file"
|
|
78
|
+
json=$(awk '/^{/{flag=1} flag' "$file")
|
|
79
|
+
echo "$json" | jq .
|
|
80
|
+
done
|
|
81
|
+
else
|
|
82
|
+
echo "No artifacts directory found."
|
|
83
|
+
fi
|
|
84
|
+
|
|
85
|
+
- name: Download baseline artifacts from main
|
|
86
|
+
if: always() && github.event_name == 'pull_request'
|
|
87
|
+
uses: dawidd6/action-download-artifact@v6
|
|
88
|
+
with:
|
|
89
|
+
workflow: qa-intelligence.yml
|
|
90
|
+
branch: ${{ github.base_ref }}
|
|
91
|
+
name: ai-failure-artifacts
|
|
92
|
+
path: baseline-artifacts
|
|
93
|
+
if_no_artifact_found: warn
|
|
94
|
+
|
|
95
|
+
- name: Debug baseline contents
|
|
96
|
+
if: always() && github.event_name == 'pull_request'
|
|
97
|
+
run: |
|
|
98
|
+
echo "Baseline:"
|
|
99
|
+
ls -la ../baseline-artifacts || true
|
|
100
|
+
echo "Current:"
|
|
101
|
+
ls -la artifacts || true
|
|
102
|
+
|
|
103
|
+
- name: Compute failure diff
|
|
104
|
+
if: always() && github.event_name == 'pull_request'
|
|
105
|
+
run: npx qa-intelligence-diff --baseline ../baseline-artifacts --current artifacts
|
|
106
|
+
|
|
107
|
+
- name: Restore failure history cache
|
|
108
|
+
if: always() && github.event_name == 'pull_request'
|
|
109
|
+
uses: actions/cache@v4
|
|
110
|
+
with:
|
|
111
|
+
path: playwright/.cache
|
|
112
|
+
key: failure-history-${{ github.repository }}-${{ github.sha }}
|
|
113
|
+
restore-keys: |
|
|
114
|
+
failure-history-${{ github.repository }}-
|
|
115
|
+
|
|
116
|
+
- name: Update failure history (recurrence)
|
|
117
|
+
if: always() && github.event_name == 'pull_request'
|
|
118
|
+
run: npx qa-intelligence-history
|
|
119
|
+
|
|
120
|
+
- name: Comment PR with AI summary
|
|
121
|
+
if: always() && github.event_name == 'pull_request'
|
|
122
|
+
env:
|
|
123
|
+
GITHUB_TOKEN: ${{ github.token }}
|
|
124
|
+
run: |
|
|
125
|
+
npx qa-intelligence-comment \
|
|
126
|
+
--diff failure-diff.json \
|
|
127
|
+
--repo ${{ github.repository }} \
|
|
128
|
+
--pr ${{ github.event.pull_request.number }} \
|
|
129
|
+
--token $GITHUB_TOKEN
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { defineConfig } from "@playwright/test";
|
|
2
|
+
import { env } from "qa-intelligence/config/env";
|
|
3
|
+
|
|
4
|
+
export default defineConfig({
|
|
5
|
+
testDir: "./tests",
|
|
6
|
+
retries: env.PW_RETRIES,
|
|
7
|
+
workers: env.PW_WORKERS,
|
|
8
|
+
globalSetup: require.resolve("qa-intelligence/playwright/globalSetup"),
|
|
9
|
+
globalTeardown: require.resolve("qa-intelligence/playwright/globalTeardown"),
|
|
10
|
+
use: {
|
|
11
|
+
baseURL: env.BASE_URL,
|
|
12
|
+
headless: env.HEADLESS,
|
|
13
|
+
trace: "on-first-retry",
|
|
14
|
+
screenshot: "only-on-failure",
|
|
15
|
+
video: "retain-on-failure",
|
|
16
|
+
},
|
|
17
|
+
reporter: [["html", { open: "never" }]],
|
|
18
|
+
});
|