scenescout 3.4.0 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +38 -12
- package/dist/engine/bench.js +485 -0
- package/dist/engine/brief.js +20 -5
- package/dist/engine/browser.js +84 -27
- package/dist/engine/calibration.js +57 -0
- package/dist/engine/claims.js +10 -3
- package/dist/engine/collector.js +17 -0
- package/dist/engine/lane.js +26 -11
- package/dist/engine/memory.js +29 -0
- package/dist/engine/oracles.js +15 -8
- package/dist/engine/pace.js +57 -11
- package/dist/engine/policy.js +40 -0
- package/dist/engine/report.js +1 -1
- package/dist/engine/request.js +14 -1
- package/dist/mcp-server.js +40 -7
- package/package.json +5 -3
- package/skills/scenescout/SKILL.md +6 -6
package/dist/engine/brief.js
CHANGED
|
@@ -92,8 +92,17 @@ export function planLanes(routes, laneCount, opts = {}) {
|
|
|
92
92
|
objective: laneObjective(lane.modules, opts.goal),
|
|
93
93
|
modules: lane.modules,
|
|
94
94
|
routes: lane.routes,
|
|
95
|
+
landing: landingOf(lane.routes),
|
|
95
96
|
}));
|
|
96
97
|
}
|
|
98
|
+
/**
|
|
99
|
+
* The first of a lane's routes a browser can open as it stands. A route the
|
|
100
|
+
* engine normalised (`/orders/:id`) is a pattern, not an address; a lane with
|
|
101
|
+
* nothing else lands on the app's root and navigates from there.
|
|
102
|
+
*/
|
|
103
|
+
export function landingOf(routes) {
|
|
104
|
+
return routes.find((r) => !/(^|\/)[:*]|\[[^\]]+\]/.test(r)) ?? "/";
|
|
105
|
+
}
|
|
97
106
|
function laneObjective(modules, goal) {
|
|
98
107
|
const owned = modules.length === 1 ? modules[0] : `${modules.slice(0, -1).join(", ")} and ${modules[modules.length - 1]}`;
|
|
99
108
|
const what = `Own ${owned}`;
|
|
@@ -114,19 +123,25 @@ export function formatBriefs(briefs, opts = {}) {
|
|
|
114
123
|
const lines = [
|
|
115
124
|
`LANE PLAN — ${briefs.length} lane(s) over ${briefs.reduce((n, b) => n + b.routes.length, 0)} route(s).`,
|
|
116
125
|
``,
|
|
117
|
-
`Give each lane its own agent. Every lane attaches with its own session name, so the browsers run genuinely in parallel:`,
|
|
118
|
-
` scout_attach { session: "<lane>", url, projectPath, mode: "${mode}"${opts.role ? `, storageStatePath: "<${opts.role}>"` : ""}, objective: "<objective>" }`,
|
|
126
|
+
`Give each lane its own agent. Every lane attaches with its own session name, so the browsers run genuinely in parallel, and lands on its own first route rather than the home page:`,
|
|
127
|
+
` scout_attach { session: "<lane>", url: "<origin><landing>", projectPath, mode: "${mode}"${opts.role ? `, storageStatePath: "<${opts.role}>"` : ""}, objective: "<objective>" }`,
|
|
119
128
|
``,
|
|
120
|
-
`Rules to pass on,
|
|
121
|
-
` · A lane works ITS routes only. Two lanes auditing the same register while a third module is never opened is the failure this plan exists to prevent — and route coverage will look complete either way.`,
|
|
129
|
+
`Rules to pass on, which a hand-written brief tends to drop:`,
|
|
130
|
+
` · A lane works ITS routes only. Two lanes auditing the same register while a third module is never opened is the failure this plan exists to prevent — and route coverage will look complete either way. A defect on a page it does not own is that page's lane's to file.`,
|
|
122
131
|
` · Every acting tool takes a \`task\`. A lane that acts with none is refused, and the person watching the live view would otherwise see a browser clicking through their app with nothing to say why.`,
|
|
123
|
-
` ·
|
|
132
|
+
` · File each defect with scout_finding the moment it is judged, before writing the report. A defect only in the report is not in the run's report.`,
|
|
133
|
+
` · Before calling a list empty, stale or stuck, read the status of the request behind it: a correct empty result and a failed one can render identically.`,
|
|
134
|
+
` · On a create form, submit one markup-shaped value, then open where that record is listed. Any lane's session catches it rendering as markup, so the list may belong to another lane.`,
|
|
135
|
+
` · When a control is withheld from this role, call the endpoint behind it with scout_request: hiding a button is not enforcing a rule. When a sort or filter runs, check the result is what it claims.`,
|
|
136
|
+
` · Check scout_coverage before finishing.`,
|
|
137
|
+
` · A lane reports back with scout_lane_report, not prose, and does NOT close its session: the planner folds the report, which checks each judged defect was filed, then closes it. Call scout_lane_report with no reply to get the instruction to put in its prompt.`,
|
|
124
138
|
``,
|
|
125
139
|
];
|
|
126
140
|
for (const b of briefs) {
|
|
127
141
|
lines.push(`── ${b.lane} ──`);
|
|
128
142
|
lines.push(`objective: ${b.objective}`);
|
|
129
143
|
lines.push(`owns: ${b.modules.join(", ")} (${b.routes.length} route(s))`);
|
|
144
|
+
lines.push(`landing: ${b.landing}`);
|
|
130
145
|
lines.push(`routes: ${b.routes.slice(0, 20).join(", ")}${b.routes.length > 20 ? ` … and ${b.routes.length - 20} more` : ""}`);
|
|
131
146
|
lines.push(``);
|
|
132
147
|
}
|
package/dist/engine/browser.js
CHANGED
|
@@ -7,7 +7,7 @@ import { normalizeTask } from "./task.js";
|
|
|
7
7
|
import { CLAIM_SCAN_SCRIPT, findContradictions } from "./claims.js";
|
|
8
8
|
import { describeInjection, newInjections, probeQueries, probeScript, probeShape, rememberProbe } from "./injection.js";
|
|
9
9
|
import { AuthLossTracker } from "./authloss.js";
|
|
10
|
-
import { COLLECT_INTERACTABLES_SCRIPT, VISIBLE_SRC, geometryIssues, BROKEN_IMAGES_SCRIPT, brokenImageIssues, } from "./collector.js";
|
|
10
|
+
import { COLLECT_INTERACTABLES_SCRIPT, VISIBLE_SRC, geometryIssues, BROKEN_IMAGES_SCRIPT, brokenImageIssues, displayName, missingName, } from "./collector.js";
|
|
11
11
|
import { OracleMonitor, formatViolations } from "./oracles.js";
|
|
12
12
|
import { extractCreatedIds, isOwnedResource, normalizeId } from "./ownership.js";
|
|
13
13
|
import { formatJourney, measureJourney } from "./journey.js";
|
|
@@ -20,7 +20,7 @@ import { explainLaunchFailure, isMissingBrowser } from "./launch.js";
|
|
|
20
20
|
import { ACTION_TIMEOUT_MS, performScroll, probeFocusIndicators, probeOverlays, scrollContainer } from "./probes.js";
|
|
21
21
|
import { BROWSER_MARKER, reapOrphanBrowsers } from "./reaper.js";
|
|
22
22
|
import { planUploadOptions, resolveDiskUpload } from "./uploads.js";
|
|
23
|
-
import { destructiveRefusal, isDestructive, isDestructiveWire, allowsWrite, isAuthExempt } from "./policy.js";
|
|
23
|
+
import { answersWithRefusal, destructiveRefusal, isDestructive, isDestructiveWire, allowsWrite, policyRefusal, isAuthExempt, } from "./policy.js";
|
|
24
24
|
import { scanProject } from "../scan.js";
|
|
25
25
|
import { analyzeDesign, DESIGN_COLLECT_SCRIPT } from "./design.js";
|
|
26
26
|
import { acceptMatches, generatedUpload } from "./fixtures.js";
|
|
@@ -206,7 +206,7 @@ export class BrowserEngine {
|
|
|
206
206
|
const list = this.newlyCreated.splice(0);
|
|
207
207
|
return `\n(created: ${list.join(", ")} — this session may edit or delete ${list.length === 1 ? "it" : "them"})`;
|
|
208
208
|
}
|
|
209
|
-
/** Design audits run this session
|
|
209
|
+
/** Design audits run by this session. The report gate counts the whole run's, on the shared store (MemoryStore.auditsThisRun). */
|
|
210
210
|
designAuditCount = 0;
|
|
211
211
|
/** Active task-efficiency measurement (scout_journey), if any. */
|
|
212
212
|
journey = null;
|
|
@@ -240,8 +240,9 @@ export class BrowserEngine {
|
|
|
240
240
|
*
|
|
241
241
|
* The fetch runs IN the page, so it passes through the same interception the
|
|
242
242
|
* write policy is enforced on: a safe-write session cannot reach past the
|
|
243
|
-
* policy by calling an endpoint instead of clicking it. The
|
|
244
|
-
*
|
|
243
|
+
* policy by calling an endpoint instead of clicking it. The policy answers
|
|
244
|
+
* the fetch with its stand-in 403, and the result says so rather than
|
|
245
|
+
* printing it as the server's status.
|
|
245
246
|
*/
|
|
246
247
|
async apiRequest(input) {
|
|
247
248
|
const page = this.requirePage();
|
|
@@ -260,23 +261,41 @@ export class BrowserEngine {
|
|
|
260
261
|
let raw;
|
|
261
262
|
try {
|
|
262
263
|
raw = (await page.evaluate(script));
|
|
264
|
+
this.forgetReplay(method.method, target.url);
|
|
263
265
|
}
|
|
264
266
|
catch (err) {
|
|
265
|
-
//
|
|
266
|
-
//
|
|
267
|
+
// The page could not run the fetch at all (a navigation mid-call, a
|
|
268
|
+
// closed page). The policy answers rather than rejects, so this is not it.
|
|
267
269
|
const message = err instanceof Error ? err.message : String(err);
|
|
268
270
|
this.logAction({ action: "request", target: `${method.method} ${input.path}`, url: page.url(), result: `blocked: ${message.split("\n")[0]}` });
|
|
269
|
-
return `${method.method} ${input.path} — the request did not complete: ${message.split("\n")[0]}
|
|
271
|
+
return `${method.method} ${input.path} — the request did not complete: ${message.split("\n")[0]}`;
|
|
270
272
|
}
|
|
271
273
|
const result = toReplayResult(raw);
|
|
272
274
|
this.logAction({
|
|
273
275
|
action: "request",
|
|
274
276
|
target: `${method.method} ${input.path}`,
|
|
275
277
|
url: page.url(),
|
|
276
|
-
|
|
278
|
+
// Not a status signature: the trail must not record the stand-in as the server's answer.
|
|
279
|
+
result: result.refusedByPolicy ? `blocked: write policy (${result.refusedByPolicy})` : replaySignature(method.method, result.url, result.status),
|
|
277
280
|
});
|
|
278
281
|
return formatReplay(method.method, result);
|
|
279
282
|
}
|
|
283
|
+
/**
|
|
284
|
+
* Take the replayed request back out of the contradiction ledger. It was the
|
|
285
|
+
* agent's call, not the page's, so whatever the page says next is not its
|
|
286
|
+
* answer — and left in, a replay the policy refused was blamed on the next
|
|
287
|
+
* click as that click's false success, in place of the click's own request.
|
|
288
|
+
* Only this request: anything else the page fetched meanwhile stays.
|
|
289
|
+
*/
|
|
290
|
+
forgetReplay(method, url) {
|
|
291
|
+
for (let i = this.watchedResponses.length - 1; i >= 0; i -= 1) {
|
|
292
|
+
const r = this.watchedResponses[i];
|
|
293
|
+
if (r.method === method && r.url === url) {
|
|
294
|
+
this.watchedResponses.splice(i, 1);
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
}
|
|
280
299
|
/**
|
|
281
300
|
* The frame field for a log entry, or nothing. Spread into logAction so a
|
|
282
301
|
* caller that is not going through afterAction can still record what the
|
|
@@ -419,7 +438,8 @@ export class BrowserEngine {
|
|
|
419
438
|
watchedResponses = [];
|
|
420
439
|
static MAX_WATCHED_RESPONSES = 60;
|
|
421
440
|
/**
|
|
422
|
-
* Requests the write policy
|
|
441
|
+
* Requests the write policy stopped, by identity — whether it dropped them or
|
|
442
|
+
* answered them with a stand-in refusal. The request event fires for
|
|
423
443
|
* every non-GET the page ATTEMPTS, before the route handler decides its fate,
|
|
424
444
|
* so without this the same DELETE was reported twice with opposite meanings:
|
|
425
445
|
* "server state may have mutated despite read-only mode" and "WRITE-POLICY
|
|
@@ -427,11 +447,27 @@ export class BrowserEngine {
|
|
|
427
447
|
* meet different fates, and a blocked /items/7/archive must not hide an
|
|
428
448
|
* allowed POST /items that shares its prefix.
|
|
429
449
|
*/
|
|
430
|
-
|
|
431
|
-
/**
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
450
|
+
refusedByPolicy = new WeakSet();
|
|
451
|
+
/**
|
|
452
|
+
* Markup-shaped values typed by any session of this run, so every later page
|
|
453
|
+
* — in this browser or another lane's — can be checked for them rendering as
|
|
454
|
+
* elements. On the shared MemoryStore, like ownership; local before attach.
|
|
455
|
+
*/
|
|
456
|
+
get probes() {
|
|
457
|
+
return this.memory?.probes ?? this.localProbes;
|
|
458
|
+
}
|
|
459
|
+
set probes(list) {
|
|
460
|
+
if (this.memory)
|
|
461
|
+
this.memory.probes = list;
|
|
462
|
+
else
|
|
463
|
+
this.localProbes = list;
|
|
464
|
+
}
|
|
465
|
+
localProbes = [];
|
|
466
|
+
/** Injections already reported, by payload and route, so a page is not reported on every snapshot — nor by a second lane that opens it. */
|
|
467
|
+
get injectionsReported() {
|
|
468
|
+
return this.memory?.injectionsReported ?? this.localInjectionsReported;
|
|
469
|
+
}
|
|
470
|
+
localInjectionsReported = new Set();
|
|
435
471
|
/**
|
|
436
472
|
* Remember a typed value when it holds an element worth watching for. What
|
|
437
473
|
* to type is the agent's choice; this only watches for the outcome. The
|
|
@@ -459,7 +495,7 @@ export class BrowserEngine {
|
|
|
459
495
|
* element on the current page? Runs wherever violations are drained, so the
|
|
460
496
|
* finding reaches the agent in the result of the action that revealed it.
|
|
461
497
|
*/
|
|
462
|
-
/** Record one answered request for the contradiction rules.
|
|
498
|
+
/** Record one answered request for the contradiction rules. Ones the policy stopped are marked, never dropped: the rules need to know they were ours. */
|
|
463
499
|
watchResponse(req, status) {
|
|
464
500
|
if (this.watchedResponses.length >= BrowserEngine.MAX_WATCHED_RESPONSES)
|
|
465
501
|
return;
|
|
@@ -468,7 +504,7 @@ export class BrowserEngine {
|
|
|
468
504
|
url: req.url(),
|
|
469
505
|
status,
|
|
470
506
|
resourceType: req.resourceType(),
|
|
471
|
-
blockedByPolicy: this.
|
|
507
|
+
blockedByPolicy: this.refusedByPolicy.has(req),
|
|
472
508
|
});
|
|
473
509
|
}
|
|
474
510
|
/**
|
|
@@ -507,18 +543,22 @@ export class BrowserEngine {
|
|
|
507
543
|
contradictionsReported = new Set();
|
|
508
544
|
async scanForInjections() {
|
|
509
545
|
const page = this.page;
|
|
510
|
-
|
|
546
|
+
// One copy for the whole scan: the list is shared with every lane, and
|
|
547
|
+
// another lane typing a payload during the await below would shift the
|
|
548
|
+
// indices the hits are matched back to.
|
|
549
|
+
const probes = this.probes;
|
|
550
|
+
if (!page || page.isClosed() || probes.length === 0)
|
|
511
551
|
return;
|
|
512
552
|
const url = page.url();
|
|
513
553
|
let hits;
|
|
514
554
|
try {
|
|
515
|
-
hits = (await page.evaluate(probeScript(probeQueries(
|
|
555
|
+
hits = (await page.evaluate(probeScript(probeQueries(probes))));
|
|
516
556
|
}
|
|
517
557
|
catch {
|
|
518
558
|
// A page mid-navigation has no DOM to ask; the next drain looks again.
|
|
519
559
|
return;
|
|
520
560
|
}
|
|
521
|
-
for (const found of newInjections(
|
|
561
|
+
for (const found of newInjections(probes, hits, url, this.injectionsReported)) {
|
|
522
562
|
this.oracles.noteInjection(describeInjection(found.probe, url, found.outer), url);
|
|
523
563
|
}
|
|
524
564
|
}
|
|
@@ -571,7 +611,7 @@ export class BrowserEngine {
|
|
|
571
611
|
this.projectDirNote = ` (its real path could not be resolved: ${err instanceof Error ? err.message : String(err)} — a symlinked project path may be wrongly refused)`;
|
|
572
612
|
}
|
|
573
613
|
this.oracles = new OracleMonitor();
|
|
574
|
-
this.oracles.
|
|
614
|
+
this.oracles.setPolicyRefusalCheck((req) => this.refusedByPolicy.has(req));
|
|
575
615
|
this.lastSnap = null;
|
|
576
616
|
this.designAuditCount = 0;
|
|
577
617
|
// The engine learns the route list itself so completion is an objective,
|
|
@@ -642,7 +682,7 @@ export class BrowserEngine {
|
|
|
642
682
|
// not just looked at — the difference between visited and tested.
|
|
643
683
|
//
|
|
644
684
|
// Only for requests the policy will actually let through. This event
|
|
645
|
-
// fires BEFORE the route handler
|
|
685
|
+
// fires BEFORE the route handler stops a blocked one, so counting it
|
|
646
686
|
// here let a REFUSED destructive POST mark the route as mutated — a form
|
|
647
687
|
// that was never submitted reading as tested, in read-only mode where by
|
|
648
688
|
// definition nothing is.
|
|
@@ -711,11 +751,16 @@ export class BrowserEngine {
|
|
|
711
751
|
}
|
|
712
752
|
return route.continue();
|
|
713
753
|
}
|
|
754
|
+
const answered = answersWithRefusal(req.resourceType());
|
|
714
755
|
if (this.blockedRequests.length < 20)
|
|
715
|
-
this.blockedRequests.push({ at: Date.now(), sig: `${method} ${url.slice(0, 140)}
|
|
756
|
+
this.blockedRequests.push({ at: Date.now(), sig: `${method} ${url.slice(0, 140)}`, answered });
|
|
716
757
|
this.logAction({ action: "write-policy:blocked", target: `${method} ${pathname}`, url: this.page?.url() ?? "" });
|
|
717
|
-
this.
|
|
758
|
+
this.refusedByPolicy.add(req);
|
|
718
759
|
this.oracles.notePolicyBlock();
|
|
760
|
+
// A script's request is answered with a refusal, so the page's handling
|
|
761
|
+
// of one actually runs; a navigation is dropped (policy.ts says why).
|
|
762
|
+
if (answered)
|
|
763
|
+
return route.fulfill(policyRefusal(this.mode, method, pathname, req.headers()["origin"]));
|
|
719
764
|
return route.abort("blockedbyclient");
|
|
720
765
|
});
|
|
721
766
|
}
|
|
@@ -1036,7 +1081,7 @@ export class BrowserEngine {
|
|
|
1036
1081
|
memory.wasExercised(fp, el.key) ? "done" : null,
|
|
1037
1082
|
el.href ? `href=${el.href.slice(0, 60)}` : null,
|
|
1038
1083
|
].filter(Boolean);
|
|
1039
|
-
return `${el.ref} ${el.role} "${el
|
|
1084
|
+
return `${el.ref} ${el.role} "${displayName(el)}"${flags.length ? ` [${flags.join(", ")}]` : ""}`;
|
|
1040
1085
|
};
|
|
1041
1086
|
// Diff mode: when re-snapshotting the same route, report only what
|
|
1042
1087
|
// changed — same idea as UI reconciliation, applied to agent context.
|
|
@@ -1254,9 +1299,13 @@ export class BrowserEngine {
|
|
|
1254
1299
|
.map((e) => this.lateMark(e))
|
|
1255
1300
|
.join("; ");
|
|
1256
1301
|
const extra = this.blockedRequests.length > 5 ? ` (+${this.blockedRequests.length - 5} more)` : "";
|
|
1302
|
+
const answered = this.blockedRequests.some((e) => e.answered);
|
|
1257
1303
|
this.blockedRequests = [];
|
|
1258
1304
|
return (`\n🛡 WRITE-POLICY blocked (${this.mode}): ${list}${extra}. ` +
|
|
1259
1305
|
`This is the tester's safety policy, NOT an app bug — do not file a finding for the resulting error UI. ` +
|
|
1306
|
+
(answered
|
|
1307
|
+
? `The page's own requests were answered with a 403 in the server's place, so the page's handling of a refusal is real: an error message is correct, and a success message is a false_success violation. `
|
|
1308
|
+
: "") +
|
|
1260
1309
|
(this.mode === "observe"
|
|
1261
1310
|
? `observe mode blocks every request that is not a GET, so no form submission reaches the server. Re-attach with mode="read-only" ONLY if the user confirms that ordinary form submissions are acceptable on this target.`
|
|
1262
1311
|
: this.mode === "read-only"
|
|
@@ -1281,7 +1330,7 @@ export class BrowserEngine {
|
|
|
1281
1330
|
// to see the DUPLICATES that the reporting dedup below intentionally hides.
|
|
1282
1331
|
this.lastActionMutationSigs = this.mutationRequests.map((e) => e.sig);
|
|
1283
1332
|
const fresh = this.mutationRequests
|
|
1284
|
-
.filter((entry) => !this.
|
|
1333
|
+
.filter((entry) => !this.refusedByPolicy.has(entry.req))
|
|
1285
1334
|
.filter((entry) => {
|
|
1286
1335
|
const key = entry.sig.split("?")[0];
|
|
1287
1336
|
if (this.reportedMutationSigs.has(key))
|
|
@@ -2009,7 +2058,7 @@ export class BrowserEngine {
|
|
|
2009
2058
|
await this.scanForContradictions();
|
|
2010
2059
|
const violations = this.oracles.drain();
|
|
2011
2060
|
const deadEnd = elements.length === 0;
|
|
2012
|
-
const unnamed = elements.filter(
|
|
2061
|
+
const unnamed = elements.filter(missingName).length;
|
|
2013
2062
|
const missingTestid = elements.filter((el) => !el.testid && !el.disabled).length;
|
|
2014
2063
|
const flags = [loginRedirect ? "AUTH-REDIRECT" : null, deadEnd ? "DEAD-END" : null, violations.length > 0 ? `${violations.length}⚠` : null].filter(Boolean);
|
|
2015
2064
|
summary.push(`${path} — ${status} · ${elements.length} el` +
|
|
@@ -2276,6 +2325,8 @@ export class BrowserEngine {
|
|
|
2276
2325
|
const payload = (await page.evaluate(DESIGN_COLLECT_SCRIPT));
|
|
2277
2326
|
payload.page.focusSamples = await probeFocusIndicators(page);
|
|
2278
2327
|
this.designAuditCount += 1;
|
|
2328
|
+
if (this.memory)
|
|
2329
|
+
this.memory.auditsThisRun += 1;
|
|
2279
2330
|
// The census is built from previous audits, so the first few pages of a run
|
|
2280
2331
|
// score with chrome included and later ones don't. That is the same warm-up
|
|
2281
2332
|
// the coverage census has: nothing is knowable as "shared" until it has been
|
|
@@ -2506,6 +2557,12 @@ export class BrowserEngine {
|
|
|
2506
2557
|
}
|
|
2507
2558
|
}
|
|
2508
2559
|
async close() {
|
|
2560
|
+
// Marks the end of the time this session held a browser, so the pace
|
|
2561
|
+
// section can say how long it was held with nothing happening. Only when a
|
|
2562
|
+
// browser is actually open: attach() closes first, and a close of nothing
|
|
2563
|
+
// is not an event.
|
|
2564
|
+
if (this.page)
|
|
2565
|
+
this.logAction({ action: "close", url: this.page.isClosed() ? "" : this.page.url() });
|
|
2509
2566
|
// Pending debounced coverage writes must land before the process can exit.
|
|
2510
2567
|
try {
|
|
2511
2568
|
this.memory?.flush();
|
|
@@ -215,3 +215,60 @@ function readEce(ece) {
|
|
|
215
215
|
return "Close enough to compare lanes by.";
|
|
216
216
|
return "Far enough apart that a lane's confidence is worth reading as an ordering rather than a rate — after checking the exclusions above, which push this upward.";
|
|
217
217
|
}
|
|
218
|
+
/** Most unfiled defects named when a lane report is folded; the rest are counted. */
|
|
219
|
+
export const MAX_UNFILED_NAMED = 10;
|
|
220
|
+
/**
|
|
221
|
+
* The defects a lane judged that no finding in the store covers yet.
|
|
222
|
+
*
|
|
223
|
+
* A lane's report is a verdict, not a filing: a defect it judged and never
|
|
224
|
+
* passed to scout_finding never reaches the report. That happened in a
|
|
225
|
+
* measured run — a covered Save button, judged a defect at 0.75 and filed by
|
|
226
|
+
* nobody — and it is cheapest to catch at the moment the planner folds the
|
|
227
|
+
* report, while the lane's session is still open.
|
|
228
|
+
*
|
|
229
|
+
* Matched on the store's failing-endpoint signature where the evidence has
|
|
230
|
+
* one, and on the evidence text otherwise. The text match is what the
|
|
231
|
+
* calibration join refuses, because there a miss is scored against the lane;
|
|
232
|
+
* here a miss only asks someone to check, which costs a look and nothing more.
|
|
233
|
+
*/
|
|
234
|
+
export function unfiledDefects(decisions, findings) {
|
|
235
|
+
const keys = new Set();
|
|
236
|
+
const texts = [];
|
|
237
|
+
for (const f of findings) {
|
|
238
|
+
if (!f.evidence)
|
|
239
|
+
continue;
|
|
240
|
+
for (const key of joinKeys(f.evidence))
|
|
241
|
+
keys.add(key);
|
|
242
|
+
texts.push(squash(f.evidence));
|
|
243
|
+
}
|
|
244
|
+
const out = [];
|
|
245
|
+
for (const d of decisions) {
|
|
246
|
+
if (d.verdict !== "defect")
|
|
247
|
+
continue;
|
|
248
|
+
if (d.evidence) {
|
|
249
|
+
const joined = [...joinKeys(d.evidence)];
|
|
250
|
+
if (joined.some((k) => keys.has(k)))
|
|
251
|
+
continue;
|
|
252
|
+
// A failing-endpoint signature is the store's own identity for a bug. When
|
|
253
|
+
// the decision has one and no finding shares it, text overlap is not a
|
|
254
|
+
// match: "GET /api/items" inside "... GET /api/items returned 500" is.
|
|
255
|
+
if (joined.length > 0) {
|
|
256
|
+
out.push(`${d.observation} — ${d.evidence}`);
|
|
257
|
+
continue;
|
|
258
|
+
}
|
|
259
|
+
const text = squash(d.evidence);
|
|
260
|
+
if (text && texts.some((t) => t === text || contains(t, text) || contains(text, t)))
|
|
261
|
+
continue;
|
|
262
|
+
}
|
|
263
|
+
out.push(d.evidence ? `${d.observation} — ${d.evidence}` : d.observation);
|
|
264
|
+
}
|
|
265
|
+
return out;
|
|
266
|
+
}
|
|
267
|
+
/** One signature inside another, when the inner one is long enough to mean something: "404" is inside half the findings in a run. */
|
|
268
|
+
function contains(outer, inner) {
|
|
269
|
+
return inner.length >= MIN_CONTAINED && outer.includes(inner);
|
|
270
|
+
}
|
|
271
|
+
const MIN_CONTAINED = 12;
|
|
272
|
+
function squash(text) {
|
|
273
|
+
return text.toLowerCase().replace(/\s+/g, " ").trim();
|
|
274
|
+
}
|
package/dist/engine/claims.js
CHANGED
|
@@ -41,7 +41,7 @@ const WRITING_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]);
|
|
|
41
41
|
const DATA_RESOURCES = new Set(["xhr", "fetch", "document"]);
|
|
42
42
|
/** Whether this request is one whose refusal the page should be admitting to. */
|
|
43
43
|
export function isRefused(req) {
|
|
44
|
-
if (req.blockedByPolicy)
|
|
44
|
+
if (req.blockedByPolicy && req.status === null)
|
|
45
45
|
return false;
|
|
46
46
|
if (!DATA_RESOURCES.has(req.resourceType))
|
|
47
47
|
return false;
|
|
@@ -100,6 +100,12 @@ function shortUrl(url) {
|
|
|
100
100
|
function say(req) {
|
|
101
101
|
return `${req.method} ${shortUrl(req.url)} ${req.status === null ? "failed" : req.status}`;
|
|
102
102
|
}
|
|
103
|
+
/** Said after a contradiction whose refusal the write policy wrote, so nobody goes looking for it in the server's logs. */
|
|
104
|
+
function standIn(req) {
|
|
105
|
+
return req.blockedByPolicy
|
|
106
|
+
? ` The refusal was the write policy's stand-in (the server never received the request); what failed is the page's handling of a refusal, which a real one would meet the same way.`
|
|
107
|
+
: "";
|
|
108
|
+
}
|
|
103
109
|
/**
|
|
104
110
|
* The contradictions this action produced, if any.
|
|
105
111
|
*
|
|
@@ -125,7 +131,8 @@ export function findContradictions(requests, page) {
|
|
|
125
131
|
out.push({
|
|
126
132
|
kind: "refused_empty",
|
|
127
133
|
detail: `${say(worst)} was refused, and the page shows ${how} with no error. ` +
|
|
128
|
-
`The user is told there is nothing to see when the truth is that nothing could be loaded
|
|
134
|
+
`The user is told there is nothing to see when the truth is that nothing could be loaded.` +
|
|
135
|
+
standIn(worst),
|
|
129
136
|
evidence: `refused-empty ${say(worst)}`,
|
|
130
137
|
});
|
|
131
138
|
}
|
|
@@ -135,7 +142,7 @@ export function findContradictions(requests, page) {
|
|
|
135
142
|
const message = page.texts[claims.indexOf("success")];
|
|
136
143
|
out.push({
|
|
137
144
|
kind: "false_success",
|
|
138
|
-
detail: `${say(worst)} was refused, and the page says ${JSON.stringify(message.trim().slice(0, 80))}. The user is told their change was kept when the server rejected it
|
|
145
|
+
detail: `${say(worst)} was refused, and the page says ${JSON.stringify(message.trim().slice(0, 80))}. The user is told their change was kept when the server rejected it.${standIn(worst)}`,
|
|
139
146
|
evidence: `false-success ${say(worst)}`,
|
|
140
147
|
});
|
|
141
148
|
}
|
package/dist/engine/collector.js
CHANGED
|
@@ -276,6 +276,23 @@ export const COLLECT_INTERACTABLES_SCRIPT = `(() => {
|
|
|
276
276
|
}
|
|
277
277
|
return out;
|
|
278
278
|
})()`;
|
|
279
|
+
/**
|
|
280
|
+
* Roles that announce content rather than take input: a live region is empty
|
|
281
|
+
* until it has something to say, and ARIA does not require it to have a name.
|
|
282
|
+
* Reporting an empty status line as an unnamed control sent lanes to file it
|
|
283
|
+
* as an accessibility defect — twice in one measured run.
|
|
284
|
+
*/
|
|
285
|
+
const LIVE_REGION_ROLES = new Set(["status", "alert", "log", "timer", "marquee"]);
|
|
286
|
+
/** Whether an element with this role and no accessible name is an unnamed control, as opposed to a live region with nothing in it yet. */
|
|
287
|
+
export function missingName(el) {
|
|
288
|
+
return !el.name && !LIVE_REGION_ROLES.has(el.role);
|
|
289
|
+
}
|
|
290
|
+
/** How the snapshot shows an element's name: an empty live region says so rather than reading as an unnamed control. */
|
|
291
|
+
export function displayName(el) {
|
|
292
|
+
if (el.name)
|
|
293
|
+
return el.name;
|
|
294
|
+
return LIVE_REGION_ROLES.has(el.role) ? "(empty live region)" : "(unnamed)";
|
|
295
|
+
}
|
|
279
296
|
/**
|
|
280
297
|
* Deterministic geometry oracles — the checks people reach for screenshots to
|
|
281
298
|
* do, computed from layout boxes instead: interactables rendered fully outside
|
package/dist/engine/lane.js
CHANGED
|
@@ -90,12 +90,21 @@ export const LaneReport = z
|
|
|
90
90
|
seen.add(d.observation);
|
|
91
91
|
});
|
|
92
92
|
});
|
|
93
|
-
const FENCE_OPEN = /^```[a-z]*\s*\n/i;
|
|
94
|
-
const FENCE_CLOSE = /\n?```\s*$/;
|
|
93
|
+
const FENCE_OPEN = /^```[a-z]*\s*\r?\n/i;
|
|
94
|
+
const FENCE_CLOSE = /\r?\n?```\s*$/;
|
|
95
|
+
/** Every fenced block in a reply, with its contents. */
|
|
96
|
+
const FENCED_BLOCK = /```[a-z]*[ \t]*\r?\n([\s\S]*?)\r?\n?```/gi;
|
|
95
97
|
/**
|
|
96
98
|
* The reply is one JSON object and nothing else. A fenced block around it is
|
|
97
|
-
* tolerated, because every model has been trained to add one
|
|
98
|
-
*
|
|
99
|
+
* tolerated, because every model has been trained to add one.
|
|
100
|
+
*
|
|
101
|
+
* Prose around ONE fenced JSON block is tolerated too, and dropped unread. The
|
|
102
|
+
* rule used to refuse it, and in a measured run six of eight lanes did it
|
|
103
|
+
* anyway: each refusal cost the planner a round trip or a hand-unwrap, to
|
|
104
|
+
* recover an object that was already unambiguous. The point — nobody has to
|
|
105
|
+
* READ the reply — survives, because the prose is never looked at. Unfenced
|
|
106
|
+
* prose is still refused, because where the object starts and ends is then a
|
|
107
|
+
* guess, and so is more than one fenced object.
|
|
99
108
|
* The reason names what was wrong, since the planner relays it to the lane.
|
|
100
109
|
*
|
|
101
110
|
* With `expectedLane`, a report that names another lane is refused: the
|
|
@@ -104,13 +113,19 @@ const FENCE_CLOSE = /\n?```\s*$/;
|
|
|
104
113
|
*/
|
|
105
114
|
export function parseLaneReport(text, expectedLane) {
|
|
106
115
|
let body = text.trim();
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
return { ok: false, reason: "the closing ``` must end the reply, with no text after it" };
|
|
116
|
+
let aroundIgnored = false;
|
|
117
|
+
if (FENCE_OPEN.test(body) && FENCE_CLOSE.test(body) && [...body.matchAll(FENCED_BLOCK)].length === 1) {
|
|
110
118
|
body = body.replace(FENCE_OPEN, "").replace(FENCE_CLOSE, "").trim();
|
|
111
119
|
}
|
|
112
|
-
if (!body.startsWith("{"))
|
|
113
|
-
|
|
120
|
+
else if (!body.startsWith("{")) {
|
|
121
|
+
const objects = [...body.matchAll(FENCED_BLOCK)].map((m) => m[1].trim()).filter((b) => b.startsWith("{"));
|
|
122
|
+
if (objects.length > 1)
|
|
123
|
+
return { ok: false, reason: `the reply holds ${objects.length} fenced JSON blocks; hand back exactly one object` };
|
|
124
|
+
if (objects.length === 0)
|
|
125
|
+
return { ok: false, reason: "the reply must be one JSON object, with no text before it" };
|
|
126
|
+
body = objects[0];
|
|
127
|
+
aroundIgnored = true;
|
|
128
|
+
}
|
|
114
129
|
let raw;
|
|
115
130
|
try {
|
|
116
131
|
raw = JSON.parse(body);
|
|
@@ -129,7 +144,7 @@ export function parseLaneReport(text, expectedLane) {
|
|
|
129
144
|
if (expectedLane !== undefined && result.data.lane !== expectedLane) {
|
|
130
145
|
return { ok: false, reason: `the report names lane "${result.data.lane}", but this reply was asked of lane "${expectedLane}"` };
|
|
131
146
|
}
|
|
132
|
-
return { ok: true, report: result.data };
|
|
147
|
+
return { ok: true, report: result.data, aroundIgnored };
|
|
133
148
|
}
|
|
134
149
|
/**
|
|
135
150
|
* The paragraph the planner puts in every lane's prompt. It is generated from
|
|
@@ -153,7 +168,7 @@ export function laneReportInstruction(lane) {
|
|
|
153
168
|
* refuses good replies.
|
|
154
169
|
*/
|
|
155
170
|
const LANE_RUBRIC = [
|
|
156
|
-
"Reply with ONE JSON object and nothing else — no prose before or after it, no explanation, no headings. A fenced ```json block is fine.",
|
|
171
|
+
"Reply with ONE JSON object and nothing else — no prose before or after it, no explanation, no headings. A fenced ```json block is fine; anything outside it is discarded unread, so put nothing there you want kept.",
|
|
157
172
|
`Shape: {"lane":<your lane name>,"status":<${quoteAll(LANE_STATUSES)}>,"decisions":[…],"routes":[…],"blocked_by":<string or null>}.`,
|
|
158
173
|
`Each decision: {"observation":<a short id for what was observed, unique in the report, at most ${LANE_OBSERVATION_MAX} characters>,"verdict":<${quoteAll(LANE_VERDICTS)}>,"severity":<${quoteAll(LANE_SEVERITIES)} or null>,"category":<${quoteAll(LANE_CATEGORIES)} or null>,"confidence":<0..1>,"evidence":<machine signature such as "GET /api/things 500", or null>}.`,
|
|
159
174
|
`A "defect" must carry a severity and a category. "evidence" is a signature, not a sentence: at most ${LANE_EVIDENCE_MAX} characters. "confidence" is how sure you are of the verdict, calibrated: 0.5 means a coin flip, 0.95 means you would bet on it.`,
|
package/dist/engine/memory.js
CHANGED
|
@@ -567,6 +567,35 @@ export class MemoryStore {
|
|
|
567
567
|
ownedIds = new Map();
|
|
568
568
|
/** Human-readable creation log for this run — becomes the report's cleanup list. */
|
|
569
569
|
createdResources = [];
|
|
570
|
+
/**
|
|
571
|
+
* Markup-shaped values any session of this run typed, and the injections
|
|
572
|
+
* already reported. Shared for the same reason ownership is: a parallel run
|
|
573
|
+
* splits the app by route, so the lane that types a payload into a create
|
|
574
|
+
* form is rarely the lane that opens the list rendering it, and a probe kept
|
|
575
|
+
* per session could only ever catch a reflected injection. Not persisted: a
|
|
576
|
+
* payload typed in an earlier run is not this run's evidence.
|
|
577
|
+
*/
|
|
578
|
+
probes = [];
|
|
579
|
+
injectionsReported = new Set();
|
|
580
|
+
/**
|
|
581
|
+
* Design audits any session of this run performed. The report's gate asked
|
|
582
|
+
* the session that happened to call scout_report, so a run whose lanes
|
|
583
|
+
* audited every page was refused because the planner's own session had not.
|
|
584
|
+
* Not persisted: an audit from an earlier run does not satisfy this one.
|
|
585
|
+
*/
|
|
586
|
+
auditsThisRun = 0;
|
|
587
|
+
/**
|
|
588
|
+
* End the run: forget what only this run's sessions know. Called when the
|
|
589
|
+
* last session on this project closes. The store itself outlives it — the
|
|
590
|
+
* server keeps one per project for the life of the process — so without
|
|
591
|
+
* this, a second run in the same process passed the audit gate on the first
|
|
592
|
+
* run's audit and stayed silent about an injection the first run reported.
|
|
593
|
+
*/
|
|
594
|
+
endRun() {
|
|
595
|
+
this.probes = [];
|
|
596
|
+
this.injectionsReported.clear();
|
|
597
|
+
this.auditsThisRun = 0;
|
|
598
|
+
}
|
|
570
599
|
constructor(projectDir) {
|
|
571
600
|
this.dir = path.join(projectDir, MEMORY_DIRNAME);
|
|
572
601
|
this.legacyDirNote = adoptLegacyMemoryDir(projectDir);
|
package/dist/engine/oracles.js
CHANGED
|
@@ -17,7 +17,8 @@ export const POLICY_BLOCK_WINDOW_MS = 2000;
|
|
|
17
17
|
* block rather than something the app did?
|
|
18
18
|
*
|
|
19
19
|
* Aborting a request makes the browser print a console error, and — when the
|
|
20
|
-
* app does not catch the rejection — raise a page error.
|
|
20
|
+
* app does not catch the rejection — raise a page error. Answering one with the
|
|
21
|
+
* policy's stand-in 403 makes the browser print its "status of 403" line. Left in, a report
|
|
21
22
|
* lists the tool's own safety net as defects of the app under test. (The
|
|
22
23
|
* failed REQUEST itself is matched exactly, by request identity, in the
|
|
23
24
|
* monitor; these two carry no request to match on, so they are attributed by
|
|
@@ -32,7 +33,7 @@ export function isPolicyInduced(v, msSincePolicyBlock) {
|
|
|
32
33
|
return false;
|
|
33
34
|
if (v.kind !== "page_error" && v.kind !== "console_error")
|
|
34
35
|
return false;
|
|
35
|
-
return /ERR_BLOCKED_BY_CLIENT|Failed to fetch|NetworkError when attempting to fetch|Load failed/i.test(v.detail);
|
|
36
|
+
return /ERR_BLOCKED_BY_CLIENT|Failed to fetch|NetworkError when attempting to fetch|Load failed|the server responded with a status of 403\b/i.test(v.detail);
|
|
36
37
|
}
|
|
37
38
|
/**
|
|
38
39
|
* Invariant oracles: passive listeners that record violations regardless of
|
|
@@ -74,7 +75,7 @@ export class OracleMonitor {
|
|
|
74
75
|
return;
|
|
75
76
|
if (BENIGN_URL_RE.test(req.url()))
|
|
76
77
|
return;
|
|
77
|
-
if (this.
|
|
78
|
+
if (this.refusedByPolicy(req)) {
|
|
78
79
|
this.policyAttributed += 1;
|
|
79
80
|
return;
|
|
80
81
|
}
|
|
@@ -93,6 +94,12 @@ export class OracleMonitor {
|
|
|
93
94
|
// reported here on every page load teaches the driver to ignore http_error.
|
|
94
95
|
if (BENIGN_URL_RE.test(res.url()))
|
|
95
96
|
return;
|
|
97
|
+
// The write policy's own stand-in refusal, matched by request identity
|
|
98
|
+
// like a dropped one: the server never said this.
|
|
99
|
+
if (this.refusedByPolicy(res.request())) {
|
|
100
|
+
this.policyAttributed += 1;
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
96
103
|
// 401/403 are often expected (auth probes); still report, but as medium.
|
|
97
104
|
this.record({
|
|
98
105
|
kind: "http_error",
|
|
@@ -113,7 +120,7 @@ export class OracleMonitor {
|
|
|
113
120
|
*/
|
|
114
121
|
static MAX_REPORTED_SIGS = 5000;
|
|
115
122
|
lastPolicyBlockAt = null;
|
|
116
|
-
|
|
123
|
+
refusedByPolicy = () => false;
|
|
117
124
|
/**
|
|
118
125
|
* Errors attributed to the write policy's own blocks and therefore not
|
|
119
126
|
* recorded as violations. Counted, so a report can say how many there were:
|
|
@@ -121,11 +128,11 @@ export class OracleMonitor {
|
|
|
121
128
|
* errors look the same.
|
|
122
129
|
*/
|
|
123
130
|
policyAttributed = 0;
|
|
124
|
-
/** The engine knows exactly which requests
|
|
125
|
-
|
|
126
|
-
this.
|
|
131
|
+
/** The engine knows exactly which requests its policy stopped; failed requests and stand-in refusals are matched against that, not against wording. */
|
|
132
|
+
setPolicyRefusalCheck(check) {
|
|
133
|
+
this.refusedByPolicy = check;
|
|
127
134
|
}
|
|
128
|
-
/** Called by the engine when the write policy
|
|
135
|
+
/** Called by the engine when the write policy stops a request (dropped or answered), so the errors that causes are not held against the app. */
|
|
129
136
|
notePolicyBlock() {
|
|
130
137
|
this.lastPolicyBlockAt = Date.now();
|
|
131
138
|
}
|