scenescout 3.11.0 → 3.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/README.md +5 -1
- package/dist/browsers.js +52 -8
- package/dist/check-run.js +4 -2
- package/dist/engine/bench.js +147 -14
- package/dist/engine/browser.js +174 -7
- package/dist/engine/calibration.js +122 -20
- package/dist/engine/check.js +86 -14
- package/dist/engine/design.js +15 -1
- package/dist/engine/fingerprint.js +69 -7
- package/dist/engine/lane.js +138 -3
- package/dist/engine/memory.js +135 -10
- package/dist/engine/report.js +37 -2
- package/dist/engine/unload.js +201 -0
- package/dist/engine/verify.js +4 -3
- package/dist/mcp-server.js +23 -9
- package/package.json +1 -1
- package/skills/scenescout/SKILL.md +4 -3
package/dist/engine/memory.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
import { normalizePath, shortHash } from "./fingerprint.js";
|
|
3
|
+
import { laneRoutePaths, normalizePath, shortHash, stripRouteQuery } from "./fingerprint.js";
|
|
4
4
|
import { isFormBookkeeping } from "./forms.js";
|
|
5
5
|
/**
|
|
6
6
|
* The kinds a finding can be. One list: scout_finding's input schema, the lane
|
|
@@ -26,6 +26,32 @@ export const FINDING_CATEGORIES = [
|
|
|
26
26
|
"missing-testid",
|
|
27
27
|
"other",
|
|
28
28
|
];
|
|
29
|
+
/**
|
|
30
|
+
* Whether a finding is in the "worth a look" tier. Anything else a file holds
|
|
31
|
+
* reads as a defect: a finding is never taken out of the defect count on a
|
|
32
|
+
* value this version does not know.
|
|
33
|
+
*/
|
|
34
|
+
export function isWorthALook(f) {
|
|
35
|
+
return f.tier === "worth_a_look";
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* The tier a finding keeps when it is found again. A defect wins: a finding
|
|
39
|
+
* someone filed as a defect is a decision about the project's convention, and
|
|
40
|
+
* a later "worth a look" for the same thing does not undo it. A worth-a-look
|
|
41
|
+
* filed again as a defect is promoted, and loses its convention. Two
|
|
42
|
+
* worth-a-looks keep the first convention unless it had none.
|
|
43
|
+
*/
|
|
44
|
+
/** Drops a finding's tier and convention in place, so a merged tier can be assigned onto it. */
|
|
45
|
+
function withoutTier(f) {
|
|
46
|
+
delete f.tier;
|
|
47
|
+
delete f.convention;
|
|
48
|
+
return f;
|
|
49
|
+
}
|
|
50
|
+
export function mergeTier(existing, incoming) {
|
|
51
|
+
if (!isWorthALook(existing) || !isWorthALook(incoming))
|
|
52
|
+
return {};
|
|
53
|
+
return { tier: "worth_a_look", convention: existing.convention || incoming.convention };
|
|
54
|
+
}
|
|
29
55
|
/**
|
|
30
56
|
* Most states a route may keep. A route accumulates one state per distinct
|
|
31
57
|
* element set, so a register with filters, tabs and paging produces dozens;
|
|
@@ -213,12 +239,19 @@ export function mergeMemory(mine, theirs) {
|
|
|
213
239
|
// the other side did not go on to re-find it as a regression.
|
|
214
240
|
const newer = f.foundAt >= other.foundAt ? f : other;
|
|
215
241
|
const older = newer === f ? other : f;
|
|
216
|
-
|
|
242
|
+
const merged = {
|
|
217
243
|
...newer,
|
|
218
244
|
runs: Math.max(f.runs, other.runs),
|
|
219
245
|
evidence: newer.evidence ?? older.evidence,
|
|
220
246
|
regressedAt: newer.regressedAt ?? older.regressedAt,
|
|
221
|
-
}
|
|
247
|
+
};
|
|
248
|
+
// The tier is not "later knowledge wins": a defect on either side is a decision
|
|
249
|
+
// about the convention, and a store still holding the worth-a-look must not undo it.
|
|
250
|
+
const tier = mergeTier(older, newer);
|
|
251
|
+
Object.assign(withoutTier(merged), tier);
|
|
252
|
+
if (!isWorthALook(merged) && isWorthALook(newer))
|
|
253
|
+
merged.severity = older.severity;
|
|
254
|
+
byId.set(f.id, merged);
|
|
222
255
|
}
|
|
223
256
|
out.findings = [...byId.values()];
|
|
224
257
|
const unionRecord = (a, b) => (a || b ? { ...(b ?? {}), ...(a ?? {}) } : undefined);
|
|
@@ -264,8 +297,34 @@ export function mergeMemory(mine, theirs) {
|
|
|
264
297
|
out.laneDecisions = [...byDecision.values()].sort((a, b) => a.at.localeCompare(b.at)).slice(-MAX_LANE_DECISIONS);
|
|
265
298
|
if (out.laneDecisions.length === 0)
|
|
266
299
|
delete out.laneDecisions;
|
|
300
|
+
// The same reason as decisions: each lane folds in its own process. A set
|
|
301
|
+
// union per lane, so merging the same document twice changes nothing.
|
|
302
|
+
out.laneRoutes = { ...(theirs.laneRoutes ?? {}) };
|
|
303
|
+
for (const [lane, routes] of Object.entries(mine.laneRoutes ?? {})) {
|
|
304
|
+
out.laneRoutes[lane] = unionRoutes(out.laneRoutes[lane] ?? [], routes);
|
|
305
|
+
}
|
|
306
|
+
if (Object.keys(out.laneRoutes).length === 0)
|
|
307
|
+
delete out.laneRoutes;
|
|
267
308
|
return out;
|
|
268
309
|
}
|
|
310
|
+
/** Most routes kept per lane. A lane report carries at most a few hundred; this keeps a long-lived project's file small. */
|
|
311
|
+
export const MAX_LANE_ROUTES = 255;
|
|
312
|
+
/**
|
|
313
|
+
* `b` added to `a`. Two routes naming the same page(s) are one — the later
|
|
314
|
+
* wording wins — and past the cap the NEWEST are kept: a long-lived project
|
|
315
|
+
* reuses lane names, and what a lane covers now is what a new verdict is
|
|
316
|
+
* judged against.
|
|
317
|
+
*/
|
|
318
|
+
function unionRoutes(a, b) {
|
|
319
|
+
const byPage = new Map();
|
|
320
|
+
for (const r of [...a, ...b]) {
|
|
321
|
+
const paths = laneRoutePaths(r);
|
|
322
|
+
const id = paths.length ? paths.sort().join(" ") : `text:${r}`;
|
|
323
|
+
byPage.delete(id);
|
|
324
|
+
byPage.set(id, r);
|
|
325
|
+
}
|
|
326
|
+
return [...byPage.values()].slice(-MAX_LANE_ROUTES);
|
|
327
|
+
}
|
|
269
328
|
/**
|
|
270
329
|
* What makes two stored decisions the same judgement.
|
|
271
330
|
*
|
|
@@ -451,6 +510,14 @@ export function requestsDisagree(a, b) {
|
|
|
451
510
|
* refused save is data-loss to one and data-inconsistency to another. Across
|
|
452
511
|
* families — a layout defect and a data defect — a shared quoted string names
|
|
453
512
|
* a place on the page, not a bug.
|
|
513
|
+
*
|
|
514
|
+
* The presentation kinds are each a family of their own. They are not
|
|
515
|
+
* alternative labels for one observation: one element is routinely wrong in
|
|
516
|
+
* several independent ways at once — clipped out of view (visual), styled like
|
|
517
|
+
* body text (ux-polish), unnamed to a screen reader (a11y), missing its test id
|
|
518
|
+
* — and the findings about it quote the same control name and name the same
|
|
519
|
+
* test id. As one family, a link "styled like body text" was merged into the
|
|
520
|
+
* finding that the same link was clipped, and the second finding was lost.
|
|
454
521
|
*/
|
|
455
522
|
const CATEGORY_FAMILY = {
|
|
456
523
|
"data-inconsistency": "data",
|
|
@@ -460,10 +527,10 @@ const CATEGORY_FAMILY = {
|
|
|
460
527
|
network: "failure",
|
|
461
528
|
"console-error": "failure",
|
|
462
529
|
"page-error": "failure",
|
|
463
|
-
visual: "
|
|
464
|
-
"ux-polish": "
|
|
465
|
-
a11y: "
|
|
466
|
-
"missing-testid": "
|
|
530
|
+
visual: "visual",
|
|
531
|
+
"ux-polish": "ux-polish",
|
|
532
|
+
a11y: "a11y",
|
|
533
|
+
"missing-testid": "missing-testid",
|
|
467
534
|
"ux-confusing": "flow",
|
|
468
535
|
"dead-end": "flow",
|
|
469
536
|
security: "security",
|
|
@@ -485,7 +552,30 @@ export function sameFamily(a, b) {
|
|
|
485
552
|
const fa = CATEGORY_FAMILY[a];
|
|
486
553
|
return fa !== undefined && fa === CATEGORY_FAMILY[b];
|
|
487
554
|
}
|
|
555
|
+
/**
|
|
556
|
+
* Whether two findings may be one defect by KIND — what is wrong, as opposed
|
|
557
|
+
* to where. The same category, or two categories of one family. Every
|
|
558
|
+
* route-scoped rule below asks this first: evidence, a quoted literal and a
|
|
559
|
+
* title all say where a defect is as often as what it is, so none of them may
|
|
560
|
+
* merge a finding of one kind into a finding of another.
|
|
561
|
+
*/
|
|
562
|
+
function sameKind(a, b) {
|
|
563
|
+
return (!!a && a === b) || sameFamily(a, b);
|
|
564
|
+
}
|
|
565
|
+
/**
|
|
566
|
+
* The categories a finding of `category` can merge with, in list order —
|
|
567
|
+
* so scout_finding can tell a lane whose filing was merged which kinds would
|
|
568
|
+
* have been kept apart. An unknown category merges only with itself.
|
|
569
|
+
*/
|
|
570
|
+
export function mergeableCategories(category) {
|
|
571
|
+
const known = FINDING_CATEGORIES.filter((c) => sameKind(category, c));
|
|
572
|
+
return known.length > 0 ? known : [category];
|
|
573
|
+
}
|
|
488
574
|
function sameFinding(a, b) {
|
|
575
|
+
// What is wrong before where: two kinds of defect are two findings, however
|
|
576
|
+
// alike their evidence, quoted text or titles.
|
|
577
|
+
if (!sameKind(a.category, b.category))
|
|
578
|
+
return false;
|
|
489
579
|
const aEv = a.evidence?.toLowerCase().replace(/\s+/g, " ").trim();
|
|
490
580
|
const bEv = b.evidence?.toLowerCase().replace(/\s+/g, " ").trim();
|
|
491
581
|
if (aEv && bEv && aEv === bEv)
|
|
@@ -506,7 +596,9 @@ function sameFinding(a, b) {
|
|
|
506
596
|
// literal with the data defect describing what the button does — in its
|
|
507
597
|
// detail or its own title. Merged, the layout defect was filed and then lost
|
|
508
598
|
// from the report on most runs of a benchmark. Within a family, one bug
|
|
509
|
-
// filed twice under neighbouring categories still merges.
|
|
599
|
+
// filed twice under neighbouring categories still merges. Stricter than the
|
|
600
|
+
// kind check above in one way: an unknown category read from an older file
|
|
601
|
+
// is never merged on a quoted string, even with itself.
|
|
510
602
|
//
|
|
511
603
|
// And never when both findings' evidence names requests with no endpoint in
|
|
512
604
|
// common. A quoted control name bridges two findings about that control
|
|
@@ -847,6 +939,11 @@ export class MemoryStore {
|
|
|
847
939
|
dupOf.evidence = f.evidence;
|
|
848
940
|
if (f.status === "resolved")
|
|
849
941
|
dupOf.status = "resolved";
|
|
942
|
+
const promoted = isWorthALook(dupOf) && !isWorthALook(f);
|
|
943
|
+
const tier = mergeTier(dupOf, f);
|
|
944
|
+
Object.assign(withoutTier(dupOf), tier);
|
|
945
|
+
if (promoted)
|
|
946
|
+
dupOf.severity = f.severity;
|
|
850
947
|
merged += 1;
|
|
851
948
|
}
|
|
852
949
|
else {
|
|
@@ -1087,6 +1184,27 @@ export class MemoryStore {
|
|
|
1087
1184
|
// every call after the first reported nothing kept — while storing fine.
|
|
1088
1185
|
return Math.min(added, MAX_LANE_DECISIONS);
|
|
1089
1186
|
}
|
|
1187
|
+
/** The routes each lane's report said it covered. Empty on a project that has never run one. */
|
|
1188
|
+
get laneRoutes() {
|
|
1189
|
+
return this.data.laneRoutes ?? {};
|
|
1190
|
+
}
|
|
1191
|
+
/**
|
|
1192
|
+
* Record the routes a lane's accepted report listed, added to any it listed
|
|
1193
|
+
* before. Redacted like every other string a lane writes, with query
|
|
1194
|
+
* strings and fragments removed. Returns how many were new.
|
|
1195
|
+
*/
|
|
1196
|
+
addLaneRoutes(lane, routes) {
|
|
1197
|
+
const before = this.data.laneRoutes?.[lane] ?? [];
|
|
1198
|
+
// No query string or fragment is stored: a route is copied from the
|
|
1199
|
+
// address bar, and an address can carry a token.
|
|
1200
|
+
const after = unionRoutes(before, routes.map((r) => redactSecrets(stripRouteQuery(r))).filter((r) => r.length > 0));
|
|
1201
|
+
const added = after.filter((r) => !before.includes(r)).length;
|
|
1202
|
+
if (after.length === before.length && after.every((r, i) => r === before[i]))
|
|
1203
|
+
return 0;
|
|
1204
|
+
this.data.laneRoutes = { ...(this.data.laneRoutes ?? {}), [lane]: after };
|
|
1205
|
+
this.flush();
|
|
1206
|
+
return added;
|
|
1207
|
+
}
|
|
1090
1208
|
/** Mark a finding resolved; returns it or null. */
|
|
1091
1209
|
resolveFinding(id) {
|
|
1092
1210
|
const f = this.data.findings.find((x) => x.id === id);
|
|
@@ -1356,6 +1474,7 @@ export class MemoryStore {
|
|
|
1356
1474
|
* across sessions rarely reuses the exact words — Jaccard catches it).
|
|
1357
1475
|
* Returns [finding, isNew].
|
|
1358
1476
|
*/
|
|
1477
|
+
/** Returns the finding, whether it is new, and whether an existing worth-a-look was just promoted to a defect by it. */
|
|
1359
1478
|
addFinding(input) {
|
|
1360
1479
|
// Redact BEFORE the id is derived, so a re-found finding whose quoted
|
|
1361
1480
|
// secret differs by a character still hashes to the same id.
|
|
@@ -1376,6 +1495,12 @@ export class MemoryStore {
|
|
|
1376
1495
|
existing.foundAt = new Date().toISOString();
|
|
1377
1496
|
if (!existing.evidence && f.evidence)
|
|
1378
1497
|
existing.evidence = f.evidence;
|
|
1498
|
+
// A worth-a-look filed again as a defect is promoted, at the severity the defect was filed at.
|
|
1499
|
+
const promoted = isWorthALook(existing) && !isWorthALook(f);
|
|
1500
|
+
const tier = mergeTier(existing, f);
|
|
1501
|
+
Object.assign(withoutTier(existing), tier);
|
|
1502
|
+
if (promoted)
|
|
1503
|
+
existing.severity = f.severity;
|
|
1379
1504
|
// Re-finding a RESOLVED finding is a regression — reopen it loudly
|
|
1380
1505
|
// rather than letting it hide in the report's completed section. But a
|
|
1381
1506
|
// FUZZY match must never resurrect a fixed bug: telling someone a
|
|
@@ -1392,7 +1517,7 @@ export class MemoryStore {
|
|
|
1392
1517
|
existing.regressedAt = existing.foundAt;
|
|
1393
1518
|
}
|
|
1394
1519
|
this.flush();
|
|
1395
|
-
return [existing, false];
|
|
1520
|
+
return [existing, false, promoted];
|
|
1396
1521
|
}
|
|
1397
1522
|
// Repro trace scoped to the finding's route: everything since the action
|
|
1398
1523
|
// that landed there, not 12 lines of unrelated cross-module noise.
|
|
@@ -1429,7 +1554,7 @@ export class MemoryStore {
|
|
|
1429
1554
|
};
|
|
1430
1555
|
this.data.findings.push(finding);
|
|
1431
1556
|
this.flush();
|
|
1432
|
-
return [finding, true];
|
|
1557
|
+
return [finding, true, false];
|
|
1433
1558
|
}
|
|
1434
1559
|
get findings() {
|
|
1435
1560
|
return this.data.findings;
|
package/dist/engine/report.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
import { SHARED_CHROME_ROUTE, isEmbedKey } from "./memory.js";
|
|
3
|
+
import { SHARED_CHROME_ROUTE, isEmbedKey, isWorthALook } from "./memory.js";
|
|
4
4
|
import { sayVerification } from "./verify.js";
|
|
5
5
|
import { feedForSession } from "./live.js";
|
|
6
6
|
import { buildReplayHtml, evidenceFor } from "./replay.js";
|
|
@@ -103,6 +103,34 @@ function projectName(dir) {
|
|
|
103
103
|
const parent = path.dirname(dir);
|
|
104
104
|
return path.basename(parent) || path.basename(dir) || "project";
|
|
105
105
|
}
|
|
106
|
+
/**
|
|
107
|
+
* The "Worth a look" section: observations that are real and are defects only
|
|
108
|
+
* under a convention of the project the run cannot see. Each says what was
|
|
109
|
+
* seen and what would confirm it. Listed below the findings and never in their
|
|
110
|
+
* totals, because SceneScout does not decide a project's conventions; nothing
|
|
111
|
+
* when there are none. The first bullet names the id, as a finding's does, so
|
|
112
|
+
* the live view hangs the frames recorded around it under the entry.
|
|
113
|
+
*/
|
|
114
|
+
export function formatWorthALook(items, sessionStart) {
|
|
115
|
+
if (items.length === 0)
|
|
116
|
+
return [];
|
|
117
|
+
const lines = [
|
|
118
|
+
`## Worth a look (${items.length})`,
|
|
119
|
+
``,
|
|
120
|
+
`Real observations that are defects only under a convention of your project that the run cannot see. They are not counted as findings above; each says what would make it one.`,
|
|
121
|
+
``,
|
|
122
|
+
];
|
|
123
|
+
for (const f of items) {
|
|
124
|
+
lines.push(`### ${f.title}`, ``);
|
|
125
|
+
lines.push(`- **Id:** \`${f.id}\` · **Category:** ${f.category}${f.foundAt >= sessionStart ? "" : " · seen in an earlier run"}`);
|
|
126
|
+
lines.push(`- **A defect only if** your project uses ${f.convention ?? "a convention the finding does not name"}`);
|
|
127
|
+
if (f.evidence)
|
|
128
|
+
lines.push(`- **Seen:** \`${f.evidence}\``);
|
|
129
|
+
lines.push(`- **Where:** \`${f.state}\` (${f.url})`);
|
|
130
|
+
lines.push(``, f.detail, ``);
|
|
131
|
+
}
|
|
132
|
+
return lines;
|
|
133
|
+
}
|
|
106
134
|
/**
|
|
107
135
|
* How long ago a finding was last seen, for the historical index. A finding
|
|
108
136
|
* nobody has re-confirmed in four months is a different thing from one seen
|
|
@@ -447,7 +475,10 @@ export function generateReport(memory, oracleLog, extras, opts = {}) {
|
|
|
447
475
|
const findings = [...memory.findings].sort((a, b) => SEVERITY_ORDER[a.severity] - SEVERITY_ORDER[b.severity]);
|
|
448
476
|
const lines = [];
|
|
449
477
|
const resolved = findings.filter((f) => f.status === "resolved");
|
|
450
|
-
|
|
478
|
+
// Worth a look is listed below the findings and counted nowhere a defect is:
|
|
479
|
+
// "open", "current" and "historical" hold defects only.
|
|
480
|
+
const worthALook = findings.filter((f) => f.status !== "resolved" && isWorthALook(f));
|
|
481
|
+
const open = findings.filter((f) => f.status !== "resolved" && !isWorthALook(f));
|
|
451
482
|
const now = Date.now();
|
|
452
483
|
const current = open.filter((f) => f.foundAt >= memory.sessionStart);
|
|
453
484
|
const historical = open.filter((f) => f.foundAt < memory.sessionStart);
|
|
@@ -460,6 +491,8 @@ export function generateReport(memory, oracleLog, extras, opts = {}) {
|
|
|
460
491
|
lines.push(`| Metric | Value |`);
|
|
461
492
|
lines.push(`|---|---|`);
|
|
462
493
|
lines.push(`| Open findings | ${open.length} (${open.filter((f) => f.severity === "high").length} high) — ${current.length} seen this session, ${historical.length} historical${resolved.length ? `, ${resolved.length} resolved (listed at the bottom)` : ""} |`);
|
|
494
|
+
if (worthALook.length > 0)
|
|
495
|
+
lines.push(`| Worth a look (not counted as defects) | ${worthALook.length} |`);
|
|
463
496
|
if (extras && extras.routesTotal > 0)
|
|
464
497
|
lines.push(`| Route coverage | ${extras.routesVisited}/${extras.routesTotal} |`);
|
|
465
498
|
lines.push(`| States explored | ${cov.states} |`);
|
|
@@ -620,6 +653,7 @@ export function generateReport(memory, oracleLog, extras, opts = {}) {
|
|
|
620
653
|
lines.push(``);
|
|
621
654
|
}
|
|
622
655
|
}
|
|
656
|
+
lines.push(...formatWorthALook(worthALook, memory.sessionStart));
|
|
623
657
|
if (resolved.length > 0) {
|
|
624
658
|
lines.push(`## ✅ Resolved (${resolved.length})`);
|
|
625
659
|
lines.push(``);
|
|
@@ -750,6 +784,7 @@ export function generateReport(memory, oracleLog, extras, opts = {}) {
|
|
|
750
784
|
`Top open findings:`,
|
|
751
785
|
...open.slice(0, 10).map((f) => ` ${SEVERITY_ICON[f.severity]} [${f.severity}] ${f.title} (${f.id})`),
|
|
752
786
|
...(open.length > 10 ? [` … +${open.length - 10} more in the report`] : []),
|
|
787
|
+
...(worthALook.length > 0 ? [`Worth a look (defects only under a project convention; not counted above): ${worthALook.length}`] : []),
|
|
753
788
|
``,
|
|
754
789
|
`Gap ledger${gaps.length === 0 ? ": EMPTY — nothing known left untested" : ` (${gaps.length}):`}`,
|
|
755
790
|
...gaps.map((g) => ` ⚠ ${g}`),
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Writes the context's route handler never sees, and how the engine judges
|
|
3
|
+
* them anyway.
|
|
4
|
+
*
|
|
5
|
+
* In Chromium, a request a page sends while it is being left (a
|
|
6
|
+
* `navigator.sendBeacon` or a `fetch(..., { keepalive: true })` on
|
|
7
|
+
* `pagehide`, `visibilitychange` or `unload`, or one still leaving as a page
|
|
8
|
+
* is closed) is never handed to the context's route handler: it arrives with
|
|
9
|
+
* no network id, or after the page's own DevTools session has gone, and the
|
|
10
|
+
* driver lets it through unseen. Left alone it reached the server in every
|
|
11
|
+
* mode, observe included. The engine therefore also intercepts at the
|
|
12
|
+
* browser level (the DevTools Fetch domain on the browser target, which does
|
|
13
|
+
* pause these requests) and judges there every write the route handler did
|
|
14
|
+
* not already judge, by the same rules (ADR 2). Which engines need this is
|
|
15
|
+
* `unloadWriteInterception` in `browsers.ts`.
|
|
16
|
+
*
|
|
17
|
+
* Everything here is pure, so the rules are table-tested in policy-test; the
|
|
18
|
+
* wiring to the browser lives in `browser.ts`.
|
|
19
|
+
*/
|
|
20
|
+
import { createHash } from "node:crypto";
|
|
21
|
+
import { allowsForeignWriteOnSignIn, allowsWrite, foreignWrite, isAuthExempt, offAppPageWrite, trustsForeignWrite } from "./policy.js";
|
|
22
|
+
/** Whether a method sends nothing the write policy judges. */
|
|
23
|
+
export function isReadMethod(method) {
|
|
24
|
+
return method === "GET" || method === "HEAD" || method === "OPTIONS";
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Who sent a write the route handler never saw, as far as its headers tell:
|
|
28
|
+
* the embed it is charged to (`foreign`) and the off-app page it came from
|
|
29
|
+
* (`offApp`), the two inputs `judgeUnseenWrite` takes from the sender.
|
|
30
|
+
*
|
|
31
|
+
* The sending frame may be gone, so its URL is not known. The page it was
|
|
32
|
+
* sent from is its Referer; where there is none, the page the session drives
|
|
33
|
+
* stands in for `offApp` only.
|
|
34
|
+
*
|
|
35
|
+
* - `foreign`: policy.ts `foreignWrite` with no frame chain, so a write out of
|
|
36
|
+
* the app whose Origin header names another site is that site's embed's. A
|
|
37
|
+
* sign-in provider's page reached as the whole page, beaconing to itself as
|
|
38
|
+
* it is left, is therefore refused outside destructive; the route handler,
|
|
39
|
+
* which knows the frame, would let it through. The cautious direction. An
|
|
40
|
+
* `Origin: null` write out of the app is taken to be an embed's too, whether
|
|
41
|
+
* or not the page embeds anything, since the page can no longer be asked.
|
|
42
|
+
* - The sign-in exception (a captcha on the app's own sign-in page) is judged
|
|
43
|
+
* against the Referer, the page the request was sent from, never against
|
|
44
|
+
* whatever page the session has moved to by the time the request is seen.
|
|
45
|
+
* With no Referer it does not apply.
|
|
46
|
+
* - Trusted embeds (safe-write only) are judged by the Origin header alone.
|
|
47
|
+
*/
|
|
48
|
+
export function unseenWriteSource(input) {
|
|
49
|
+
const { appUrl, mode, url, originHeader, referer } = input;
|
|
50
|
+
let foreign = foreignWrite(appUrl, { url, frameChain: [], frameUrl: null, originHeader, pageHasForeignFrame: true });
|
|
51
|
+
if (foreign && referer && allowsForeignWriteOnSignIn(mode, referer, appUrl))
|
|
52
|
+
foreign = null;
|
|
53
|
+
if (foreign && trustsForeignWrite(mode, input.trustedEmbeds, appUrl, { frameChain: [], originHeader, unadoptedPageUrl: null }))
|
|
54
|
+
foreign = null;
|
|
55
|
+
const offApp = offAppPageWrite(appUrl, referer ?? input.currentPageUrl, url, input.movedByEmbed);
|
|
56
|
+
return { foreign, offApp };
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* The route handler's verdict on a write, from what a browser-level
|
|
60
|
+
* interception can know about it. Same rules in the same order: a write
|
|
61
|
+
* another site's frame sends outside the app is refused; so is one sent from
|
|
62
|
+
* a page an embed moved off the app, a sign-in excepted; a sign-in passes;
|
|
63
|
+
* then the mode decides. `foreign` and `offApp` come from
|
|
64
|
+
* `unseenWriteSource`, since the sending frame may no longer exist.
|
|
65
|
+
* The one step it does not take is safe-write's short wait for a creation
|
|
66
|
+
* still in flight: a page being left has nothing more to wait for.
|
|
67
|
+
*/
|
|
68
|
+
export function judgeUnseenWrite(input) {
|
|
69
|
+
const { mode, method, pathname, destructiveWire } = input;
|
|
70
|
+
if (mode === "destructive" || isReadMethod(method))
|
|
71
|
+
return { allow: true };
|
|
72
|
+
if (input.foreign)
|
|
73
|
+
return { allow: false, why: `sent from a frame of ${input.foreign}` };
|
|
74
|
+
const authExempt = isAuthExempt(mode, method, pathname, destructiveWire);
|
|
75
|
+
if (input.offApp && !authExempt)
|
|
76
|
+
return { allow: false, why: `sent from a page of ${input.offApp}, outside the app` };
|
|
77
|
+
if (authExempt)
|
|
78
|
+
return { allow: true };
|
|
79
|
+
return allowsWrite(mode, method, destructiveWire, input.owned) ? { allow: true } : { allow: false };
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* How long a write the route handler let through stays on the ledger. The
|
|
83
|
+
* browser-level pause follows the route handler's `continue` within
|
|
84
|
+
* milliseconds; an entry that outlives this is a request that never reached
|
|
85
|
+
* the network (answered by a service worker, cancelled by the page).
|
|
86
|
+
*/
|
|
87
|
+
export const ROUTED_WRITE_TTL_MS = 5000;
|
|
88
|
+
/** Entries past this many are dropped oldest first: a page that posts in a loop cannot grow the ledger without bound. */
|
|
89
|
+
const MAX_ROUTED_WRITES = 200;
|
|
90
|
+
/** A request body's identity for the ledger: the SHA-256 of its bytes, or "" for no body. */
|
|
91
|
+
export function bodyDigest(body) {
|
|
92
|
+
return body && body.length > 0 ? createHash("sha256").update(body).digest("hex") : "";
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* The writes the route handler has already judged and let through, so the
|
|
96
|
+
* browser-level interception, which sees them again, lets them pass without
|
|
97
|
+
* judging them twice. The key is the mode, the method, the URL and the body's
|
|
98
|
+
* digest (`bodyDigest`), so a write let through can only excuse the very same
|
|
99
|
+
* request: never one with another body to the same URL, such as a delete
|
|
100
|
+
* command sent where a harmless save was allowed a moment before, and never
|
|
101
|
+
* one sent after a flow switched to a stricter rule. A write is claimed once:
|
|
102
|
+
* two identical requests, one routed and one not, leave the second to be
|
|
103
|
+
* judged. Requests the route handler refused never reach the browser level,
|
|
104
|
+
* so they are not recorded.
|
|
105
|
+
*/
|
|
106
|
+
export class RoutedWrites {
|
|
107
|
+
ttlMs;
|
|
108
|
+
entries = [];
|
|
109
|
+
constructor(ttlMs = ROUTED_WRITE_TTL_MS) {
|
|
110
|
+
this.ttlMs = ttlMs;
|
|
111
|
+
}
|
|
112
|
+
static key(mode, method, url, digest) {
|
|
113
|
+
return `${mode} ${method} ${url.split("#")[0]} ${digest}`;
|
|
114
|
+
}
|
|
115
|
+
prune(now) {
|
|
116
|
+
this.entries = this.entries.filter((e) => now - e.at <= this.ttlMs);
|
|
117
|
+
if (this.entries.length > MAX_ROUTED_WRITES)
|
|
118
|
+
this.entries = this.entries.slice(-MAX_ROUTED_WRITES);
|
|
119
|
+
}
|
|
120
|
+
note(mode, method, url, digest, now = Date.now()) {
|
|
121
|
+
this.prune(now);
|
|
122
|
+
this.entries.push({ key: RoutedWrites.key(mode, method, url, digest), at: now });
|
|
123
|
+
}
|
|
124
|
+
/** Whether the route handler let this write through a moment ago; each note is claimed at most once. */
|
|
125
|
+
claim(mode, method, url, digest, now = Date.now()) {
|
|
126
|
+
this.prune(now);
|
|
127
|
+
const key = RoutedWrites.key(mode, method, url, digest);
|
|
128
|
+
const i = this.entries.findIndex((e) => e.key === key);
|
|
129
|
+
if (i < 0)
|
|
130
|
+
return false;
|
|
131
|
+
this.entries.splice(i, 1);
|
|
132
|
+
return true;
|
|
133
|
+
}
|
|
134
|
+
clear() {
|
|
135
|
+
this.entries = [];
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
/** How long a write refused at the browser level stays recognisable to the driver's request events. */
|
|
139
|
+
export const UNSEEN_REFUSAL_TTL_MS = 10000;
|
|
140
|
+
/**
|
|
141
|
+
* Writes refused at the browser level, which the driver may still report as
|
|
142
|
+
* requests of its own: a 307 or 308 hop is a request Playwright knows, and it
|
|
143
|
+
* then sees it fail (ERR_BLOCKED_BY_CLIENT). Without this the refusal was also
|
|
144
|
+
* filed as the app's failed request and listed as a possible mutation. Keyed
|
|
145
|
+
* by method and URL (fragment dropped), since the driver's request object is
|
|
146
|
+
* not the paused one; kept briefly, and not consumed, since several listeners
|
|
147
|
+
* ask about the same request.
|
|
148
|
+
*/
|
|
149
|
+
export class UnseenRefusals {
|
|
150
|
+
ttlMs;
|
|
151
|
+
entries = [];
|
|
152
|
+
constructor(ttlMs = UNSEEN_REFUSAL_TTL_MS) {
|
|
153
|
+
this.ttlMs = ttlMs;
|
|
154
|
+
}
|
|
155
|
+
static key(method, url) {
|
|
156
|
+
return `${method} ${url.split("#")[0]}`;
|
|
157
|
+
}
|
|
158
|
+
prune(now) {
|
|
159
|
+
this.entries = this.entries.filter((e) => now - e.at <= this.ttlMs);
|
|
160
|
+
if (this.entries.length > MAX_ROUTED_WRITES)
|
|
161
|
+
this.entries = this.entries.slice(-MAX_ROUTED_WRITES);
|
|
162
|
+
}
|
|
163
|
+
note(method, url, now = Date.now()) {
|
|
164
|
+
this.prune(now);
|
|
165
|
+
this.entries.push({ key: UnseenRefusals.key(method, url), at: now });
|
|
166
|
+
}
|
|
167
|
+
has(method, url, now = Date.now()) {
|
|
168
|
+
this.prune(now);
|
|
169
|
+
const key = UnseenRefusals.key(method, url);
|
|
170
|
+
return this.entries.some((e) => e.key === key);
|
|
171
|
+
}
|
|
172
|
+
clear() {
|
|
173
|
+
this.entries = [];
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* The body of a request paused by the DevTools Fetch domain, as bytes: the
|
|
178
|
+
* base64 entries decoded and joined, else the plain `postData`. Undefined when
|
|
179
|
+
* the protocol sent neither (a body too large to include). Its digest then
|
|
180
|
+
* matches no routed write, so the request is judged, on its URL alone.
|
|
181
|
+
*/
|
|
182
|
+
export function pausedRequestBytes(request) {
|
|
183
|
+
const entries = request.postDataEntries?.filter((e) => typeof e.bytes === "string");
|
|
184
|
+
if (entries && entries.length > 0)
|
|
185
|
+
return Buffer.concat(entries.map((e) => Buffer.from(e.bytes, "base64")));
|
|
186
|
+
return request.postData === undefined ? undefined : Buffer.from(request.postData, "utf8");
|
|
187
|
+
}
|
|
188
|
+
/** The same body as text, for the policy's body rules. */
|
|
189
|
+
export function pausedRequestBody(request) {
|
|
190
|
+
return pausedRequestBytes(request)?.toString("utf8");
|
|
191
|
+
}
|
|
192
|
+
/** A header's value from a DevTools header map, whose names keep the case they were sent with. */
|
|
193
|
+
export function headerOf(headers, name) {
|
|
194
|
+
if (!headers)
|
|
195
|
+
return undefined;
|
|
196
|
+
const wanted = name.toLowerCase();
|
|
197
|
+
for (const [k, v] of Object.entries(headers))
|
|
198
|
+
if (k.toLowerCase() === wanted)
|
|
199
|
+
return v;
|
|
200
|
+
return undefined;
|
|
201
|
+
}
|
package/dist/engine/verify.js
CHANGED
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
* Pure, so the ordering and the wording can be table-tested.
|
|
24
24
|
*/
|
|
25
25
|
import { normalizePath } from "./fingerprint.js";
|
|
26
|
-
import { failingSignatures } from "./memory.js";
|
|
26
|
+
import { failingSignatures, isWorthALook } from "./memory.js";
|
|
27
27
|
/** What a re-test found. */
|
|
28
28
|
export const VERDICTS = ["gone", "present", "changed"];
|
|
29
29
|
export const VERDICT_MEANING = {
|
|
@@ -230,10 +230,11 @@ export function retestResults(candidates, pages) {
|
|
|
230
230
|
/**
|
|
231
231
|
* The findings a check re-tests: the open ones it can reproduce by loading a
|
|
232
232
|
* page, in the campaign's order and within its cap, plus how many open
|
|
233
|
-
* findings there are in all, so the report can say how many it left to a run.
|
|
233
|
+
* findings (defects, not worth-a-look) there are in all, so the report can say how many it left to a run.
|
|
234
234
|
*/
|
|
235
235
|
export function checkRetestPlan(findings) {
|
|
236
|
-
|
|
236
|
+
// A worth-a-look is not a defect, so a check never re-tests it and it can never gate.
|
|
237
|
+
const open = findings.filter((f) => (f.status ?? "open") !== "resolved" && !isWorthALook(f));
|
|
237
238
|
const eligible = open.filter((f) => loadRetest(item(f)) !== null);
|
|
238
239
|
const candidates = verifyWorklist(eligible)
|
|
239
240
|
.map(loadRetest)
|
package/dist/mcp-server.js
CHANGED
|
@@ -34,8 +34,8 @@ import { ErrorCode, GetPromptRequestSchema, ListPromptsRequestSchema, McpError }
|
|
|
34
34
|
import { z } from "zod";
|
|
35
35
|
import { BrowserEngine } from "./engine/browser.js";
|
|
36
36
|
import { reapOrphanBrowsers } from "./engine/reaper.js";
|
|
37
|
-
import { FINDING_CATEGORIES, MemoryStore, redactSecrets } from "./engine/memory.js";
|
|
38
|
-
import { decodedEntitiesNote, LANE_NAME_MAX, LaneLedger, laneCloseGuard, laneReportInstruction, parseLaneReport, summarizeLaneReport } from "./engine/lane.js";
|
|
37
|
+
import { FINDING_CATEGORIES, isWorthALook, MemoryStore, mergeableCategories, redactSecrets } from "./engine/memory.js";
|
|
38
|
+
import { decodedEntitiesNote, ignoredConventionsNote, LANE_CONVENTION_MAX, LANE_NAME_MAX, LaneLedger, laneCloseGuard, laneReportInstruction, parseLaneReport, summarizeLaneReport, } from "./engine/lane.js";
|
|
39
39
|
import { MAX_UNFILED_NAMED, unfiledDefects } from "./engine/calibration.js";
|
|
40
40
|
import { SessionQueue, withWatchdog } from "./engine/dispatch.js";
|
|
41
41
|
import { FIXTURE_KINDS } from "./engine/fixtures.js";
|
|
@@ -562,6 +562,9 @@ server.registerTool("scout_lane_report", {
|
|
|
562
562
|
const kept = owner
|
|
563
563
|
? owner.addLaneDecisions(lane, parsed.report.decisions.map((d) => ({ ...d, lane, at })))
|
|
564
564
|
: 0;
|
|
565
|
+
// And the routes it covered, so the benchmark can tell a lane's remark
|
|
566
|
+
// about another lane's page from a verdict on its own.
|
|
567
|
+
owner?.addLaneRoutes(lane, parsed.report.routes);
|
|
565
568
|
// Say when nothing was kept. Every lane closing its session before the
|
|
566
569
|
// planner folds its report is the order the method describes, and it
|
|
567
570
|
// leaves no memory to write to — reporting a bare "accepted" while the
|
|
@@ -586,7 +589,7 @@ server.registerTool("scout_lane_report", {
|
|
|
586
589
|
: "";
|
|
587
590
|
laneLedger.fold(lane, engines.get(lane)?.attached === true);
|
|
588
591
|
const around = parsed.aroundIgnored ? `\n(The text around the report's JSON block was discarded unread.)` : "";
|
|
589
|
-
const decoded = decodedEntitiesNote(parsed.entitiesDecoded);
|
|
592
|
+
const decoded = decodedEntitiesNote(parsed.entitiesDecoded) + ignoredConventionsNote(parsed.conventionsIgnored);
|
|
590
593
|
return {
|
|
591
594
|
content: [{ type: "text", text: `Lane report accepted — ${summarizeLaneReport(parsed.report)}${note}${around}${decoded}${followUp}` }],
|
|
592
595
|
};
|
|
@@ -1147,28 +1150,39 @@ server.registerTool("scout_finding", {
|
|
|
1147
1150
|
.string()
|
|
1148
1151
|
.optional()
|
|
1149
1152
|
.describe("Canonical machine signature for dedup, e.g. 'GET /api/reports/dashboard 403' or 'widget dashboard-summary-widget shows 0'. Same bug re-found later should produce the same string."),
|
|
1153
|
+
convention: z
|
|
1154
|
+
.string()
|
|
1155
|
+
.min(1)
|
|
1156
|
+
.max(LANE_CONVENTION_MAX)
|
|
1157
|
+
.optional()
|
|
1158
|
+
.describe("Only for a WORTH-A-LOOK finding: the observation is real, and it is a defect only under a convention of this project you cannot see. Name that convention, e.g. 'a 4px spacing scale' or 'test ids on every control'. The report lists it under \"Worth a look\", apart from the defects, and does not count it as one. Not for \"I could not tell\": leave that unfiled or look closer. Omit for a defect."),
|
|
1150
1159
|
session: sessionParam,
|
|
1151
1160
|
},
|
|
1152
|
-
}, serializedPerSession("scout_finding", async ({ severity, category, title, detail, evidence, }, session) => {
|
|
1161
|
+
}, serializedPerSession("scout_finding", async ({ severity, category, title, detail, evidence, convention, }, session) => {
|
|
1153
1162
|
try {
|
|
1154
1163
|
const eng = engineFor(session);
|
|
1155
1164
|
if (!eng.memory)
|
|
1156
1165
|
throw new Error("Not attached — findings need an active session.");
|
|
1157
|
-
const [finding, isNew] = eng.memory.addFinding({
|
|
1166
|
+
const [finding, isNew, promoted] = eng.memory.addFinding({
|
|
1158
1167
|
severity,
|
|
1159
1168
|
category: category,
|
|
1160
1169
|
title,
|
|
1161
1170
|
detail,
|
|
1162
1171
|
evidence,
|
|
1172
|
+
...(convention ? { tier: "worth_a_look", convention } : {}),
|
|
1163
1173
|
url: eng.currentUrl,
|
|
1164
1174
|
state: eng.currentState || "(unknown)",
|
|
1165
1175
|
session: eng.sessionKey,
|
|
1166
1176
|
});
|
|
1167
1177
|
return text(isNew
|
|
1168
|
-
?
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1178
|
+
? isWorthALook(finding)
|
|
1179
|
+
? `Recorded as worth a look (not a defect in the report's totals): ${finding.title} (id ${finding.id}) — a defect only if your project uses ${finding.convention}`
|
|
1180
|
+
: `Finding recorded: [${finding.severity}] ${finding.title} (id ${finding.id})`
|
|
1181
|
+
: promoted
|
|
1182
|
+
? `Merged into finding ${finding.id}, which was worth a look, and promoted to a defect: [${finding.severity}] ${finding.title}. It now counts among the report's findings.`
|
|
1183
|
+
: finding.regressedAt
|
|
1184
|
+
? `⟳ REOPENED as a REGRESSION: finding ${finding.id} was previously resolved but the evidence reproduces again (seen in ${finding.runs} runs). Worth calling out to the user.`
|
|
1185
|
+
: `Not recorded as new: merged into existing finding ${finding.id} — [${finding.severity}] ${finding.title}${finding.evidence ? ` (evidence: ${finding.evidence.slice(0, 160)})` : " (no evidence)"}, filed as ${finding.category}, seen in ${finding.runs} runs. If yours is a different bug, file it again: under the category that says what is wrong if it is another kind of defect (a finding filed as ${category} merges only with one filed as ${mergeableCategories(category).join(" or ")}), or with evidence naming the request that failed for you (method and path) — two findings are kept apart when both name requests and none is shared.`, session);
|
|
1172
1186
|
}
|
|
1173
1187
|
catch (err) {
|
|
1174
1188
|
return errorText(err);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "scenescout",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.12.0",
|
|
4
4
|
"description": "SceneScout — exploratory UI testing for AI coding agents. An MCP server that gives any agent (Claude Code, Cursor, VS Code Copilot, Codex, Gemini CLI and others) a structured view of a running web app, always-on oracles, a network-level write policy, memory across runs and a gap-checked report.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "brunoboto96",
|