tripwire-e2e 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +201 -3
- package/bin/tripwire.js +8 -0
- package/operators/_util.js +127 -0
- package/operators/agent-proposed.js +41 -0
- package/operators/auth-drop.js +30 -0
- package/operators/dead-control.js +23 -0
- package/operators/double-apply.js +20 -0
- package/operators/fake-success.js +21 -0
- package/operators/field-swap.js +42 -0
- package/operators/hide-control.js +30 -0
- package/operators/index.js +81 -0
- package/operators/item-drop.js +36 -0
- package/operators/item-dup.js +20 -0
- package/operators/label-swap.js +30 -0
- package/operators/missing-row-action.js +24 -0
- package/operators/number-drift.js +40 -0
- package/operators/rendered-item-drop.js +31 -0
- package/operators/rendered-item-dup.js +20 -0
- package/operators/rendered-number-drift.js +24 -0
- package/operators/silent-fail.js +19 -0
- package/operators/stale-read.js +34 -0
- package/operators/text-change.js +43 -0
- package/operators/validation-off.js +21 -0
- package/operators/wrong-link.js +28 -0
- package/operators/wrong-owner.js +41 -0
- package/package.json +41 -4
- package/policies/tripwire.clean.policy.yaml +23 -0
- package/policies/tripwire.mutant.policy.yaml +13 -0
- package/src/actions.js +59 -0
- package/src/agent.js +30 -0
- package/src/browser/mutate.js +156 -0
- package/src/browser/recorder.js +102 -0
- package/src/cli.js +41 -0
- package/src/commands/check.js +45 -0
- package/src/commands/coverage.js +110 -0
- package/src/commands/doctor.js +28 -0
- package/src/commands/fix.js +208 -0
- package/src/commands/list.js +24 -0
- package/src/commands/proxy.js +28 -0
- package/src/commands/report.js +11 -0
- package/src/commands/run.js +370 -0
- package/src/commands/sabotage.js +85 -0
- package/src/config.js +52 -0
- package/src/jsonpath.js +58 -0
- package/src/kane.js +158 -0
- package/src/ledger.js +53 -0
- package/src/mutant.js +26 -0
- package/src/orphans.js +40 -0
- package/src/profile.js +46 -0
- package/src/proxy.js +292 -0
- package/src/report.js +183 -0
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
// The operator registry and mutant selection.
|
|
2
|
+
import numberDrift from './number-drift.js';
|
|
3
|
+
import itemDrop from './item-drop.js';
|
|
4
|
+
import itemDup from './item-dup.js';
|
|
5
|
+
import fieldSwap from './field-swap.js';
|
|
6
|
+
import staleRead from './stale-read.js';
|
|
7
|
+
import wrongOwner from './wrong-owner.js';
|
|
8
|
+
import renderedNumberDrift from './rendered-number-drift.js';
|
|
9
|
+
import renderedItemDrop from './rendered-item-drop.js';
|
|
10
|
+
import renderedItemDup from './rendered-item-dup.js';
|
|
11
|
+
import silentFail from './silent-fail.js';
|
|
12
|
+
import fakeSuccess from './fake-success.js';
|
|
13
|
+
import doubleApply from './double-apply.js';
|
|
14
|
+
import validationOff from './validation-off.js';
|
|
15
|
+
import authDrop from './auth-drop.js';
|
|
16
|
+
import hideControl from './hide-control.js';
|
|
17
|
+
import deadControl from './dead-control.js';
|
|
18
|
+
import labelSwap from './label-swap.js';
|
|
19
|
+
import textChange from './text-change.js';
|
|
20
|
+
import wrongLink from './wrong-link.js';
|
|
21
|
+
import missingRowAction from './missing-row-action.js';
|
|
22
|
+
import agentProposed from './agent-proposed.js';
|
|
23
|
+
|
|
24
|
+
export const OPERATORS = [
|
|
25
|
+
numberDrift, itemDrop, itemDup, fieldSwap, staleRead, wrongOwner, renderedNumberDrift, renderedItemDrop, renderedItemDup,
|
|
26
|
+
silentFail, fakeSuccess, doubleApply, validationOff, authDrop,
|
|
27
|
+
hideControl, deadControl, labelSwap, textChange, wrongLink, missingRowAction,
|
|
28
|
+
agentProposed
|
|
29
|
+
];
|
|
30
|
+
|
|
31
|
+
// Generate every candidate, merge duplicates (same id from several tests), drop excluded equivalents.
|
|
32
|
+
export function generateAll(profile, { dir = '.tripwire', only, excluded = new Set() } = {}) {
|
|
33
|
+
const byId = new Map();
|
|
34
|
+
const counts = {};
|
|
35
|
+
for (const op of OPERATORS) {
|
|
36
|
+
if (only && !only.includes(op.name) && !only.includes(op.family)) continue;
|
|
37
|
+
let made = [];
|
|
38
|
+
try { made = op.generate(profile, { dir }); } catch (err) { made = []; counts[op.name] = { error: err.message }; continue; }
|
|
39
|
+
counts[op.name] = { generated: made.length };
|
|
40
|
+
for (const m of made) {
|
|
41
|
+
if (!m.touches_tests.length) continue;
|
|
42
|
+
if (byId.has(m.id)) {
|
|
43
|
+
const prev = byId.get(m.id);
|
|
44
|
+
prev.touches_tests = [...new Set([...prev.touches_tests, ...m.touches_tests])].sort();
|
|
45
|
+
} else byId.set(m.id, m);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
const all = [...byId.values()];
|
|
49
|
+
return { candidates: all.filter((m) => !excluded.has(m.id)), excluded: all.filter((m) => excluded.has(m.id)), counts };
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// Pick up to max mutants, spread across families, operators and tests (round robin), saboteur first.
|
|
53
|
+
export function select(candidates, max) {
|
|
54
|
+
if (!max || candidates.length <= max) return candidates;
|
|
55
|
+
const groups = new Map();
|
|
56
|
+
for (const m of candidates) {
|
|
57
|
+
const k = m.operator;
|
|
58
|
+
if (!groups.has(k)) groups.set(k, []);
|
|
59
|
+
groups.get(k).push(m);
|
|
60
|
+
}
|
|
61
|
+
// within an operator, alternate between the tests that are touched so no single test dominates
|
|
62
|
+
for (const [k, list] of groups) {
|
|
63
|
+
const byTest = new Map();
|
|
64
|
+
for (const m of list) { const t = m.touches_tests[0]; if (!byTest.has(t)) byTest.set(t, []); byTest.get(t).push(m); }
|
|
65
|
+
const queues = [...byTest.values()];
|
|
66
|
+
const spread = [];
|
|
67
|
+
while (queues.some((q) => q.length)) for (const q of queues) if (q.length) spread.push(q.shift());
|
|
68
|
+
groups.set(k, spread);
|
|
69
|
+
}
|
|
70
|
+
// Saboteur mutants are domain rules written from the acceptance criteria: all of them run first.
|
|
71
|
+
const picked = (groups.get('agent-proposed') || []).splice(0, max);
|
|
72
|
+
const order = [...groups.keys()].filter((k) => k !== 'agent-proposed');
|
|
73
|
+
while (picked.length < max && order.some((k) => groups.get(k).length)) {
|
|
74
|
+
for (const k of order) {
|
|
75
|
+
if (picked.length >= max) break;
|
|
76
|
+
const g = groups.get(k);
|
|
77
|
+
if (g.length) picked.push(g.shift());
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return picked;
|
|
81
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
// Data: one element is removed from a list response.
|
|
2
|
+
import { jsonReads, walk, rendered, testText, touchingRoute, mutant, nice, lastKey } from './_util.js';
|
|
3
|
+
|
|
4
|
+
export function lists(profile) {
|
|
5
|
+
const out = [];
|
|
6
|
+
for (const t of profile.tests) {
|
|
7
|
+
const text = testText(t);
|
|
8
|
+
for (const e of jsonReads(t)) {
|
|
9
|
+
walk(e.json, (p, v) => {
|
|
10
|
+
if (!Array.isArray(v) || v.length === 0 || typeof v[0] !== 'object' || v[0] === null) return;
|
|
11
|
+
const label = Object.entries(v[0]).find(([k, x]) => typeof x === 'string' && rendered(x, text, k));
|
|
12
|
+
if (label) out.push({ e, p, v, label: label[1] });
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
return out;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
const dropOp = {
|
|
20
|
+
name: 'item-drop',
|
|
21
|
+
family: 'data',
|
|
22
|
+
summary: 'One element is removed from a list response.',
|
|
23
|
+
generate(profile) {
|
|
24
|
+
return lists(profile).map(({ e, p, v, label }) => mutant({
|
|
25
|
+
id: `item-drop:${e.method} ${e.path}:${p}[0]`,
|
|
26
|
+
family: 'data', operator: 'item-drop',
|
|
27
|
+
match: { method: e.method, path: e.path },
|
|
28
|
+
action: { type: 'json', ops: [{ op: 'remove-index', path: p, index: 0 }] },
|
|
29
|
+
touches_tests: touchingRoute(profile, e.method, e.path),
|
|
30
|
+
describe: `"${label}" is missing from the ${nice(lastKey(p) || 'list')} (${v.length} → ${v.length - 1} items)`,
|
|
31
|
+
expect: 'The list length or a named item is checked'
|
|
32
|
+
}));
|
|
33
|
+
}
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
export default dropOp;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
// Data: one element of a list response is repeated.
|
|
2
|
+
import { touchingRoute, mutant, nice, lastKey } from './_util.js';
|
|
3
|
+
import { lists } from './item-drop.js';
|
|
4
|
+
|
|
5
|
+
export default {
|
|
6
|
+
name: 'item-dup',
|
|
7
|
+
family: 'data',
|
|
8
|
+
summary: 'One element of a list response appears twice.',
|
|
9
|
+
generate(profile) {
|
|
10
|
+
return lists(profile).map(({ e, p, v, label }) => mutant({
|
|
11
|
+
id: `item-dup:${e.method} ${e.path}:${p}[0]`,
|
|
12
|
+
family: 'data', operator: 'item-dup',
|
|
13
|
+
match: { method: e.method, path: e.path },
|
|
14
|
+
action: { type: 'json', ops: [{ op: 'dup-index', path: p, index: 0 }] },
|
|
15
|
+
touches_tests: touchingRoute(profile, e.method, e.path),
|
|
16
|
+
describe: `"${label}" appears twice in the ${nice(lastKey(p) || 'list')} (${v.length} → ${v.length + 1} items)`,
|
|
17
|
+
expect: 'The list length or uniqueness is checked'
|
|
18
|
+
}));
|
|
19
|
+
}
|
|
20
|
+
};
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
// Interface: two button labels on the same page are exchanged (Save/Cancel, Approve/Reject).
|
|
2
|
+
import { controls, mutant, targetOf } from './_util.js';
|
|
3
|
+
import { domMutant } from './hide-control.js';
|
|
4
|
+
|
|
5
|
+
export default {
|
|
6
|
+
name: 'label-swap',
|
|
7
|
+
family: 'interface',
|
|
8
|
+
summary: 'Two button labels on the same page are exchanged.',
|
|
9
|
+
generate(profile) {
|
|
10
|
+
const buttons = controls(profile).filter((c) => c.role === 'button' && c.text.length <= 40 && !c.pattern && !c.text.startsWith('['));
|
|
11
|
+
const out = new Map();
|
|
12
|
+
for (const a of buttons) {
|
|
13
|
+
for (const b of buttons) {
|
|
14
|
+
if (a.text >= b.text) continue;
|
|
15
|
+
if (!a.pages.some((p) => b.pages.includes(p))) continue;
|
|
16
|
+
if (!a.clicked && !b.clicked) continue; // swapping two untouched labels is near-equivalent
|
|
17
|
+
const id = `label-swap:"${a.text}"<->"${b.text}"`;
|
|
18
|
+
out.set(id, mutant({
|
|
19
|
+
id, family: 'interface', operator: 'label-swap',
|
|
20
|
+
match: { document: true },
|
|
21
|
+
action: domMutant('swap-labels', a, { other: targetOf(b) }),
|
|
22
|
+
touches_tests: [...a.tests, ...b.tests],
|
|
23
|
+
describe: `The "${a.text}" and "${b.text}" buttons have each other's labels`,
|
|
24
|
+
expect: 'The test checks the outcome of the button it pressed'
|
|
25
|
+
}));
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
return [...out.values()].slice(0, 6);
|
|
29
|
+
}
|
|
30
|
+
};
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// Interface: the edit/delete/cancel action is missing from one row of a list or table.
|
|
2
|
+
import { controls, mutant, labelOf } from './_util.js';
|
|
3
|
+
import { domMutant } from './hide-control.js';
|
|
4
|
+
|
|
5
|
+
export default {
|
|
6
|
+
name: 'missing-row-action',
|
|
7
|
+
family: 'interface',
|
|
8
|
+
summary: 'The delete or edit action is removed from one row.',
|
|
9
|
+
generate(profile) {
|
|
10
|
+
// Row actions are buttons, or links that read like an action (a product name link is data).
|
|
11
|
+
const ACTION = /^(edit|delete|remove|cancel|view|open|details|archive|approve|reject|undo|rename|manage)\b/i;
|
|
12
|
+
return controls(profile)
|
|
13
|
+
.filter((c) => c.inRow && (c.role === 'button' || (c.role === 'link' && ACTION.test(c.text))) && c.text.length <= 30)
|
|
14
|
+
.map((c) => mutant({
|
|
15
|
+
id: `missing-row-action:"${c.text}"`,
|
|
16
|
+
family: 'interface', operator: 'missing-row-action',
|
|
17
|
+
match: { document: true },
|
|
18
|
+
action: domMutant('remove-row-action', c),
|
|
19
|
+
touches_tests: c.tests,
|
|
20
|
+
describe: `The first row loses ${labelOf({ ...c, row: undefined })}`,
|
|
21
|
+
expect: `The test checks that each row offers "${c.text}"`
|
|
22
|
+
}));
|
|
23
|
+
}
|
|
24
|
+
};
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
// Data: a price, total or count in a JSON response is off by one unit or by 10%.
|
|
2
|
+
import { jsonReads, walk, rendered, testText, touchingRoute, mutant, lastKey, isMoneyKey, isCountKey, fmtVal, nice } from './_util.js';
|
|
3
|
+
|
|
4
|
+
export default {
|
|
5
|
+
name: 'number-drift',
|
|
6
|
+
family: 'data',
|
|
7
|
+
summary: 'A price, total or count in a JSON response is changed by one unit or 10%.',
|
|
8
|
+
generate(profile) {
|
|
9
|
+
const out = [];
|
|
10
|
+
for (const t of profile.tests) {
|
|
11
|
+
const text = testText(t);
|
|
12
|
+
for (const e of jsonReads(t)) {
|
|
13
|
+
const found = [];
|
|
14
|
+
walk(e.json, (p, v) => {
|
|
15
|
+
if (typeof v !== 'number') return;
|
|
16
|
+
const key = lastKey(p);
|
|
17
|
+
if (!rendered(v, text, key)) return;
|
|
18
|
+
// totals matter most to users, then other money, then counts
|
|
19
|
+
const score = /total/i.test(key) && !/line/i.test(key) ? 3 : isMoneyKey(key) ? 2 : isCountKey(key) ? 1 : 0;
|
|
20
|
+
found.push({ p, v, key, score });
|
|
21
|
+
});
|
|
22
|
+
found.sort((a, b) => b.score - a.score);
|
|
23
|
+
for (const { p, v, key } of found.slice(0, 4)) {
|
|
24
|
+
const op = Number.isInteger(v) && !isMoneyKey(key) ? { op: 'add', path: p, value: 1 } : { op: 'scale', path: p, value: 1.1 };
|
|
25
|
+
const after = op.op === 'add' ? v + 1 : Math.round(v * 110) / 100;
|
|
26
|
+
out.push(mutant({
|
|
27
|
+
id: `number-drift:${e.method} ${e.path}:${p}`,
|
|
28
|
+
family: 'data', operator: 'number-drift',
|
|
29
|
+
match: { method: e.method, path: e.path },
|
|
30
|
+
action: { type: 'json', ops: [op] },
|
|
31
|
+
touches_tests: touchingRoute(profile, e.method, e.path),
|
|
32
|
+
describe: `${nice(key)} from ${e.path} shows ${fmtVal(after)} instead of ${fmtVal(v)}`,
|
|
33
|
+
expect: `The displayed ${nice(key)} is checked against the expected value`
|
|
34
|
+
}));
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return out;
|
|
39
|
+
}
|
|
40
|
+
};
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// Data, in the page: a list row the user saw disappears. The in-page form of item-drop, for apps
|
|
2
|
+
// whose lists never travel as JSON (local state, server-rendered pages).
|
|
3
|
+
import { controls, mutant, targetOf } from './_util.js';
|
|
4
|
+
|
|
5
|
+
export function rowLabels(profile) {
|
|
6
|
+
const seen = new Set();
|
|
7
|
+
return controls(profile).filter((c) => {
|
|
8
|
+
if (!c.inRow || !(c.role === 'text' || c.role === 'heading') || c.pattern) return false;
|
|
9
|
+
if (!/[A-Za-z]{2}/.test(c.text) || c.text.startsWith('[') || c.text.length > 60) return false;
|
|
10
|
+
if (seen.has(c.text)) return false;
|
|
11
|
+
seen.add(c.text);
|
|
12
|
+
return true;
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export default {
|
|
17
|
+
name: 'rendered-item-drop',
|
|
18
|
+
family: 'data',
|
|
19
|
+
summary: 'A list row the user saw disappears from the page.',
|
|
20
|
+
generate(profile) {
|
|
21
|
+
return rowLabels(profile).slice(0, 4).map((c) => mutant({
|
|
22
|
+
id: `rendered-item-drop:"${c.text}"`,
|
|
23
|
+
family: 'data', operator: 'rendered-item-drop',
|
|
24
|
+
match: { document: true },
|
|
25
|
+
action: { type: 'dom', dom: { op: 'hide-row', target: targetOf(c) } },
|
|
26
|
+
touches_tests: c.tests,
|
|
27
|
+
describe: `The "${c.text}" row is missing from the list`,
|
|
28
|
+
expect: 'The test checks which rows are listed, or how many'
|
|
29
|
+
}));
|
|
30
|
+
}
|
|
31
|
+
};
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
// Data, in the page: a list row the user saw is shown twice. The in-page form of item-dup.
|
|
2
|
+
import { mutant, targetOf } from './_util.js';
|
|
3
|
+
import { rowLabels } from './rendered-item-drop.js';
|
|
4
|
+
|
|
5
|
+
export default {
|
|
6
|
+
name: 'rendered-item-dup',
|
|
7
|
+
family: 'data',
|
|
8
|
+
summary: 'A list row the user saw appears twice on the page.',
|
|
9
|
+
generate(profile) {
|
|
10
|
+
return rowLabels(profile).slice(0, 4).map((c) => mutant({
|
|
11
|
+
id: `rendered-item-dup:"${c.text}"`,
|
|
12
|
+
family: 'data', operator: 'rendered-item-dup',
|
|
13
|
+
match: { document: true },
|
|
14
|
+
action: { type: 'dom', dom: { op: 'dup-row', target: targetOf(c) } },
|
|
15
|
+
touches_tests: c.tests,
|
|
16
|
+
describe: `The "${c.text}" row appears twice in the list`,
|
|
17
|
+
expect: 'The test checks how many rows are listed'
|
|
18
|
+
}));
|
|
19
|
+
}
|
|
20
|
+
};
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// Data, in the page: a number the user sees (a count, an amount) is off by one, changed in the
|
|
2
|
+
// rendered page itself. Covers apps whose data never travels as JSON: server-rendered pages,
|
|
3
|
+
// local state, websockets. On JSON apps it complements number-drift.
|
|
4
|
+
import { controls, mutant, targetOf } from './_util.js';
|
|
5
|
+
|
|
6
|
+
export default {
|
|
7
|
+
name: 'rendered-number-drift',
|
|
8
|
+
family: 'data',
|
|
9
|
+
summary: 'A number shown on the page (a count, an amount) is off by one, changed in the page itself.',
|
|
10
|
+
generate(profile) {
|
|
11
|
+
return controls(profile)
|
|
12
|
+
.filter((c) => c.role === 'text' && (c.pattern || /\d/.test(c.text)) && c.text.length <= 40)
|
|
13
|
+
.slice(0, 8)
|
|
14
|
+
.map((c) => mutant({
|
|
15
|
+
id: `rendered-number-drift:"${c.text}"`,
|
|
16
|
+
family: 'data', operator: 'rendered-number-drift',
|
|
17
|
+
match: { document: true },
|
|
18
|
+
action: { type: 'dom', dom: { op: 'drift-number', target: targetOf(c) } },
|
|
19
|
+
touches_tests: c.tests,
|
|
20
|
+
describe: `The number in "${c.ctx || c.text}" is off by one`,
|
|
21
|
+
expect: 'The test checks the exact number shown, not just that the element is there'
|
|
22
|
+
}));
|
|
23
|
+
}
|
|
24
|
+
};
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// Behaviour: a write returns 500 and is dropped. A test that never checks the outcome passes anyway.
|
|
2
|
+
import { writes, touchingRoute, mutant } from './_util.js';
|
|
3
|
+
|
|
4
|
+
export default {
|
|
5
|
+
name: 'silent-fail',
|
|
6
|
+
family: 'behaviour',
|
|
7
|
+
summary: 'A POST, PUT, PATCH or DELETE returns 500 and the request is dropped.',
|
|
8
|
+
generate(profile) {
|
|
9
|
+
return profile.tests.flatMap((t) => writes(t).filter((e) => e.status < 400).map((e) => mutant({
|
|
10
|
+
id: `silent-fail:${e.method} ${e.path}`,
|
|
11
|
+
family: 'behaviour', operator: 'silent-fail',
|
|
12
|
+
match: { method: e.method, path: e.path },
|
|
13
|
+
action: { type: 'respond', status: 500, body: { error: 'Internal Server Error' } },
|
|
14
|
+
touches_tests: touchingRoute(profile, e.method, e.path),
|
|
15
|
+
describe: `${e.method} ${e.path} fails with a 500 and nothing is saved`,
|
|
16
|
+
expect: 'The test checks the result of the action, not only that it clicked'
|
|
17
|
+
})));
|
|
18
|
+
}
|
|
19
|
+
};
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
// Data: a read after a write returns the body from before the write.
|
|
2
|
+
import { WRITE, touchingRoute, mutant } from './_util.js';
|
|
3
|
+
|
|
4
|
+
export default {
|
|
5
|
+
name: 'stale-read',
|
|
6
|
+
family: 'data',
|
|
7
|
+
summary: 'A GET after a write returns the previous body, so the page does not show the change just made.',
|
|
8
|
+
generate(profile) {
|
|
9
|
+
const out = new Map();
|
|
10
|
+
for (const t of profile.tests) {
|
|
11
|
+
const seen = new Set();
|
|
12
|
+
let wroteAfter = new Set();
|
|
13
|
+
for (const e of t.traffic) {
|
|
14
|
+
if (e.type !== 'json' && e.method === 'GET') continue;
|
|
15
|
+
if (WRITE.has(e.method) && e.status < 400) { wroteAfter = new Set(seen); continue; }
|
|
16
|
+
if (e.method === 'GET' && e.status < 400) {
|
|
17
|
+
if (wroteAfter.has(e.path) && !out.has(e.path)) {
|
|
18
|
+
out.set(e.path, mutant({
|
|
19
|
+
id: `stale-read:GET ${e.path}`,
|
|
20
|
+
family: 'data', operator: 'stale-read',
|
|
21
|
+
match: { method: 'GET', path: e.path },
|
|
22
|
+
action: { type: 'stale-read' },
|
|
23
|
+
touches_tests: touchingRoute(profile, 'GET', e.path),
|
|
24
|
+
describe: `${e.path} keeps returning what it returned before the last change`,
|
|
25
|
+
expect: 'The test checks that the change it just made is visible'
|
|
26
|
+
}));
|
|
27
|
+
}
|
|
28
|
+
seen.add(e.path);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return [...out.values()];
|
|
33
|
+
}
|
|
34
|
+
};
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
// Interface: a heading, status badge or confirmation message shows the wrong words.
|
|
2
|
+
// The replacement is another real text of the same kind from the same app, so it is plausible.
|
|
3
|
+
import { controls, mutant } from './_util.js';
|
|
4
|
+
import { domMutant } from './hide-control.js';
|
|
5
|
+
|
|
6
|
+
export default {
|
|
7
|
+
name: 'text-change',
|
|
8
|
+
family: 'interface',
|
|
9
|
+
summary: 'A heading, status badge or confirmation message is replaced.',
|
|
10
|
+
generate(profile) {
|
|
11
|
+
const all = controls(profile);
|
|
12
|
+
// Page and section headings, not the titles of cards in a list (those are data).
|
|
13
|
+
const headings = all.filter((c) => c.role === 'heading' && !c.inRow && !c.pattern);
|
|
14
|
+
// Word statuses ("Placed", "Cancelled"); numeric badges are counts and belong to number-drift.
|
|
15
|
+
const statuses = all.filter((c) => c.role === 'text' && /status|badge|tag/.test(c.cls || '') && c.text.length <= 30 && /[A-Za-z]/.test(c.text) && !c.pattern);
|
|
16
|
+
const out = [];
|
|
17
|
+
const pick = (pool, c, fallback) => (pool.find((o) => o.text !== c.text && !o.pages.some((p) => c.pages.includes(p))) || pool.find((o) => o.text !== c.text))?.text || fallback;
|
|
18
|
+
for (const c of statuses) {
|
|
19
|
+
out.push(mutant({
|
|
20
|
+
id: `text-change:status:"${c.text}"`,
|
|
21
|
+
family: 'interface', operator: 'text-change',
|
|
22
|
+
match: { document: true },
|
|
23
|
+
action: domMutant('replace-text', c, { replacement: pick(statuses, c, 'Pending') }),
|
|
24
|
+
touches_tests: c.tests,
|
|
25
|
+
describe: `The "${c.text}" status shows "${pick(statuses, c, 'Pending')}"`,
|
|
26
|
+
expect: 'The test checks the status shown, not only that a row exists'
|
|
27
|
+
}));
|
|
28
|
+
}
|
|
29
|
+
for (const c of headings.filter((h) => h.text.length <= 60)) {
|
|
30
|
+
const replacement = pick(headings, c, 'Something went wrong');
|
|
31
|
+
out.push(mutant({
|
|
32
|
+
id: `text-change:heading:"${c.text}"`,
|
|
33
|
+
family: 'interface', operator: 'text-change',
|
|
34
|
+
match: { document: true },
|
|
35
|
+
action: domMutant('replace-text', c, { replacement }),
|
|
36
|
+
touches_tests: c.tests,
|
|
37
|
+
describe: `The heading "${c.text}" reads "${replacement}"`,
|
|
38
|
+
expect: 'The test checks it reached the right page or message'
|
|
39
|
+
}));
|
|
40
|
+
}
|
|
41
|
+
return out;
|
|
42
|
+
}
|
|
43
|
+
};
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// Behaviour: a 400 or 422 validation error is turned into 200, so invalid input looks accepted.
|
|
2
|
+
// It only fires when a test actually submits invalid input; if none does, the mutant is reported
|
|
3
|
+
// as "not reached", which is itself a finding: no test covers validation on that route.
|
|
4
|
+
import { writes, touchingRoute, mutant } from './_util.js';
|
|
5
|
+
|
|
6
|
+
export default {
|
|
7
|
+
name: 'validation-off',
|
|
8
|
+
family: 'behaviour',
|
|
9
|
+
summary: 'A 400 or 422 response is turned into 200. Invalid input appears to be accepted.',
|
|
10
|
+
generate(profile) {
|
|
11
|
+
return profile.tests.flatMap((t) => writes(t).filter((e) => e.method !== 'DELETE').map((e) => mutant({
|
|
12
|
+
id: `validation-off:${e.method} ${e.path}`,
|
|
13
|
+
family: 'behaviour', operator: 'validation-off',
|
|
14
|
+
match: { method: e.method, path: e.path },
|
|
15
|
+
action: { type: 'status-map', from: [400, 422], to: 200 },
|
|
16
|
+
touches_tests: touchingRoute(profile, e.method, e.path),
|
|
17
|
+
describe: `Invalid input to ${e.method} ${e.path} is treated as a success`,
|
|
18
|
+
expect: 'A test submits invalid input and checks the error message'
|
|
19
|
+
})));
|
|
20
|
+
}
|
|
21
|
+
};
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
// Interface: a navigation link points to a different page of the same app.
|
|
2
|
+
import { controls, mutant } from './_util.js';
|
|
3
|
+
import { domMutant } from './hide-control.js';
|
|
4
|
+
|
|
5
|
+
export default {
|
|
6
|
+
name: 'wrong-link',
|
|
7
|
+
family: 'interface',
|
|
8
|
+
summary: 'A navigation link points to a different page.',
|
|
9
|
+
generate(profile) {
|
|
10
|
+
const links = controls(profile).filter((c) => c.role === 'link' && c.href && !/^(mailto:|tel:|javascript:)/.test(c.href));
|
|
11
|
+
const hrefs = [...new Set(links.map((l) => l.href))];
|
|
12
|
+
const out = [];
|
|
13
|
+
for (const l of links.filter((x) => x.clicked)) {
|
|
14
|
+
const other = hrefs.find((h) => h !== l.href && h.split('/').length === l.href.split('/').length) || hrefs.find((h) => h !== l.href);
|
|
15
|
+
if (!other) continue;
|
|
16
|
+
out.push(mutant({
|
|
17
|
+
id: `wrong-link:"${l.text}"->${other}`,
|
|
18
|
+
family: 'interface', operator: 'wrong-link',
|
|
19
|
+
match: { document: true },
|
|
20
|
+
action: domMutant('relink', l, { href: other }),
|
|
21
|
+
touches_tests: l.tests,
|
|
22
|
+
describe: `The "${l.text}" link goes to ${other} instead of ${l.href}`,
|
|
23
|
+
expect: 'After following a link the test checks where it landed'
|
|
24
|
+
}));
|
|
25
|
+
}
|
|
26
|
+
return out;
|
|
27
|
+
}
|
|
28
|
+
};
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
// Data: a list response gains a record that belongs to someone else.
|
|
2
|
+
import { jsonReads, walk, touchingRoute, mutant, nice, lastKey } from './_util.js';
|
|
3
|
+
|
|
4
|
+
const OWNER = /^(owner|ownerId|owner_id|userId|user_id|user|customerId|customer_id|accountId|account_id|author|authorId|createdBy|created_by|tenant|tenantId)$/;
|
|
5
|
+
const OWNER_NAME = /^(ownerName|owner_name|userName|user_name|customer|customerName|customer_name|authorName|name)$/;
|
|
6
|
+
const ID = /^(id|_id|uuid|number|orderNumber|ref|reference)$/i;
|
|
7
|
+
|
|
8
|
+
export default {
|
|
9
|
+
name: 'wrong-owner',
|
|
10
|
+
family: 'data',
|
|
11
|
+
summary: "A list response gains a record marked as another user's.",
|
|
12
|
+
generate(profile) {
|
|
13
|
+
const out = new Map();
|
|
14
|
+
for (const t of profile.tests) {
|
|
15
|
+
for (const e of jsonReads(t)) {
|
|
16
|
+
walk(e.json, (p, v) => {
|
|
17
|
+
if (!Array.isArray(v) || !v.length || typeof v[0] !== 'object' || v[0] === null) return;
|
|
18
|
+
const ownerKey = Object.keys(v[0]).find((k) => OWNER.test(k));
|
|
19
|
+
if (!ownerKey) return;
|
|
20
|
+
const intruder = JSON.parse(JSON.stringify(v[0]));
|
|
21
|
+
intruder[ownerKey] = typeof intruder[ownerKey] === 'number' ? intruder[ownerKey] + 9000 : 'tw-other-user';
|
|
22
|
+
for (const k of Object.keys(intruder)) {
|
|
23
|
+
if (ID.test(k)) intruder[k] = typeof intruder[k] === 'number' ? intruder[k] + 9000 : `9${String(intruder[k]).replace(/\D/g, '') || '999'}`;
|
|
24
|
+
else if (OWNER_NAME.test(k) && typeof intruder[k] === 'string') intruder[k] = 'Jordan Okafor';
|
|
25
|
+
}
|
|
26
|
+
const id = `wrong-owner:${e.method} ${e.path}:${p}`;
|
|
27
|
+
if (out.has(id)) return;
|
|
28
|
+
out.set(id, mutant({
|
|
29
|
+
id, family: 'data', operator: 'wrong-owner',
|
|
30
|
+
match: { method: e.method, path: e.path },
|
|
31
|
+
action: { type: 'json', ops: [{ op: 'push', path: p, value: intruder, position: 'start' }] },
|
|
32
|
+
touches_tests: touchingRoute(profile, e.method, e.path),
|
|
33
|
+
describe: `Another user's record (${ownerKey}=${JSON.stringify(intruder[ownerKey])}) appears in the ${nice(lastKey(p) || 'list')}`,
|
|
34
|
+
expect: 'The test checks exactly which records are listed'
|
|
35
|
+
}));
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return [...out.values()];
|
|
40
|
+
}
|
|
41
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,43 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "tripwire-e2e",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.2.0",
|
|
4
|
+
"description": "Mutation testing for Kane CLI browser suites: a proxy breaks the app one realistic way at a time, Kane replays the suite, and survivors go to your coding agent.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"bin": {
|
|
7
|
+
"tripwire": "bin/tripwire.js"
|
|
8
|
+
},
|
|
9
|
+
"files": [
|
|
10
|
+
"bin",
|
|
11
|
+
"src",
|
|
12
|
+
"operators",
|
|
13
|
+
"policies",
|
|
14
|
+
"README.md",
|
|
15
|
+
"LICENSE"
|
|
16
|
+
],
|
|
17
|
+
"dependencies": {
|
|
18
|
+
"adm-zip": "^0.5.16",
|
|
19
|
+
"yaml": "^2.6.1"
|
|
20
|
+
},
|
|
21
|
+
"engines": {
|
|
22
|
+
"node": ">=18"
|
|
23
|
+
},
|
|
24
|
+
"keywords": [
|
|
25
|
+
"kane-cli",
|
|
26
|
+
"mutation-testing",
|
|
27
|
+
"e2e",
|
|
28
|
+
"testing",
|
|
29
|
+
"ai-agents",
|
|
30
|
+
"chaos"
|
|
31
|
+
],
|
|
32
|
+
"license": "MIT",
|
|
33
|
+
"repository": {
|
|
34
|
+
"type": "git",
|
|
35
|
+
"url": "git+https://github.com/da-b0s/tripwire.git",
|
|
36
|
+
"directory": "packages/cli"
|
|
37
|
+
},
|
|
38
|
+
"homepage": "https://github.com/da-b0s/tripwire#readme",
|
|
39
|
+
"bugs": {
|
|
40
|
+
"url": "https://github.com/da-b0s/tripwire/issues"
|
|
41
|
+
},
|
|
42
|
+
"author": "da-b0s"
|
|
43
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Judged against the clean (unmutated) baseline: the normal gate a healthy suite should clear.
|
|
2
|
+
policy: tripwire-clean-suite
|
|
3
|
+
gates:
|
|
4
|
+
- id: no-failed
|
|
5
|
+
metric: totals.failed
|
|
6
|
+
equals: 0
|
|
7
|
+
severity: blocking
|
|
8
|
+
message: The baseline must be green before mutants mean anything.
|
|
9
|
+
- id: no-broken
|
|
10
|
+
metric: totals.broken
|
|
11
|
+
equals: 0
|
|
12
|
+
severity: blocking
|
|
13
|
+
message: A broken test (its check could not be evaluated) cannot be trusted as a baseline.
|
|
14
|
+
- id: every-test-checks
|
|
15
|
+
metric: metrics.tests_without_checks
|
|
16
|
+
equals: 0
|
|
17
|
+
severity: blocking
|
|
18
|
+
message: A test with no assertion cannot kill any mutant.
|
|
19
|
+
- id: no-healed
|
|
20
|
+
metric: metrics.heal.healed_tests
|
|
21
|
+
max: 0
|
|
22
|
+
severity: warning
|
|
23
|
+
message: A healed baseline test means the recording no longer matches the app.
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Judged against the evidence pack of a run made while one mutant was armed.
|
|
2
|
+
# Inverted on purpose: a pack where every test passed means the suite walked past a real bug,
|
|
3
|
+
# so Kane's own policy engine returns a failing verdict for it.
|
|
4
|
+
#
|
|
5
|
+
# "Noticed" counts failed AND broken tests. When a replay assertion fails, Kane triages it; if the
|
|
6
|
+
# triage calls it an automation bug (e.g. "locator rot" because the page text changed), the pack
|
|
7
|
+
# records the test as broken, not failed. For a mutant run that still means the suite noticed.
|
|
8
|
+
policy: tripwire-mutant-must-be-caught
|
|
9
|
+
gates:
|
|
10
|
+
- id: someone-noticed
|
|
11
|
+
expr: "totals.failed + totals.broken >= 1"
|
|
12
|
+
severity: blocking
|
|
13
|
+
message: Every test passed while the app was broken.
|
package/src/actions.js
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
// Applies a mutant's response-body action. Pure functions, unit-tested.
|
|
2
|
+
import { locate } from './jsonpath.js';
|
|
3
|
+
|
|
4
|
+
const clone = (v) => JSON.parse(JSON.stringify(v));
|
|
5
|
+
|
|
6
|
+
// Apply a list of JSON ops to a parsed body. Returns { body, changed }.
|
|
7
|
+
export function applyJsonOps(body, ops) {
|
|
8
|
+
const out = clone(body);
|
|
9
|
+
let changed = false;
|
|
10
|
+
for (const op of ops) {
|
|
11
|
+
const at = locate(out, op.path);
|
|
12
|
+
if (!at) continue;
|
|
13
|
+
const { parent, key } = at;
|
|
14
|
+
const cur = parent[key];
|
|
15
|
+
switch (op.op) {
|
|
16
|
+
case 'add': // number drift by a fixed amount
|
|
17
|
+
if (typeof cur === 'number') { parent[key] = round(cur + op.value); changed = true; }
|
|
18
|
+
break;
|
|
19
|
+
case 'scale': // number drift by a factor (1.1 = +10%)
|
|
20
|
+
if (typeof cur === 'number') { parent[key] = round(cur * op.value); changed = true; }
|
|
21
|
+
break;
|
|
22
|
+
case 'set':
|
|
23
|
+
parent[key] = op.value; changed = true;
|
|
24
|
+
break;
|
|
25
|
+
case 'remove-index':
|
|
26
|
+
if (Array.isArray(cur) && cur.length > op.index) { cur.splice(op.index, 1); changed = true; }
|
|
27
|
+
break;
|
|
28
|
+
case 'dup-index':
|
|
29
|
+
if (Array.isArray(cur) && cur.length > op.index) { cur.splice(op.index + 1, 0, clone(cur[op.index])); changed = true; }
|
|
30
|
+
break;
|
|
31
|
+
case 'swap': { // swap two sibling string fields: path is the object, a/b are keys
|
|
32
|
+
if (cur && typeof cur === 'object' && op.a in cur && op.b in cur) {
|
|
33
|
+
[cur[op.a], cur[op.b]] = [cur[op.b], cur[op.a]]; changed = true;
|
|
34
|
+
}
|
|
35
|
+
break;
|
|
36
|
+
}
|
|
37
|
+
case 'push': // add a record to a list (wrong-owner)
|
|
38
|
+
if (Array.isArray(cur)) { op.position === 'start' ? cur.unshift(clone(op.value)) : cur.push(clone(op.value)); changed = true; }
|
|
39
|
+
break;
|
|
40
|
+
default:
|
|
41
|
+
throw new Error(`unknown json op: ${op.op}`);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return { body: out, changed };
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const round = (n) => Math.round(n * 100) / 100;
|
|
48
|
+
|
|
49
|
+
// Insert markup into an HTML document just before </head> (or at the start if absent).
|
|
50
|
+
export function injectHtml(html, snippet, where = 'head') {
|
|
51
|
+
const tag = where === 'head' ? /<\/head\s*>/i : /<\/body\s*>/i;
|
|
52
|
+
if (tag.test(html)) return html.replace(tag, (m) => snippet + m);
|
|
53
|
+
return snippet + html;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function replaceHtml(html, find, replace) {
|
|
57
|
+
if (!html.includes(find)) return { html, changed: false };
|
|
58
|
+
return { html: html.split(find).join(replace), changed: true };
|
|
59
|
+
}
|