scenescout 3.11.0 → 3.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/README.md +5 -1
- package/dist/browsers.js +52 -8
- package/dist/check-run.js +4 -2
- package/dist/engine/bench.js +147 -14
- package/dist/engine/browser.js +174 -7
- package/dist/engine/calibration.js +122 -20
- package/dist/engine/check.js +86 -14
- package/dist/engine/design.js +15 -1
- package/dist/engine/fingerprint.js +69 -7
- package/dist/engine/lane.js +138 -3
- package/dist/engine/memory.js +135 -10
- package/dist/engine/report.js +37 -2
- package/dist/engine/unload.js +201 -0
- package/dist/engine/verify.js +4 -3
- package/dist/mcp-server.js +23 -9
- package/package.json +1 -1
- package/skills/scenescout/SKILL.md +4 -3
package/dist/engine/browser.js
CHANGED
|
@@ -15,7 +15,7 @@ import { describeStep, FLOW_AFTER_LAST_STEP_MS, FLOW_STEP_TIMEOUT_MS, isAction,
|
|
|
15
15
|
import { framePath, RECORD_MAX_FRAMES } from "./replay.js";
|
|
16
16
|
import { keepWatchingUrl, normalizePace, SETTLE_TICK_MS, shouldKeepWaiting } from "./settle.js";
|
|
17
17
|
import { buildRequestScript, formatReplay, replaySignature, requestHeaders, resolveMethod, resolveRequestUrl, toReplayResult } from "./request.js";
|
|
18
|
-
import { defaultEngine, focusAdvanceKey, REMOVE_SHARED_WORKER_SCRIPT, screencastSupport, serviceWorkerPolicy, sharedWorkersAllowed, } from "../browsers.js";
|
|
18
|
+
import { defaultEngine, focusAdvanceKey, REMOVE_SHARED_WORKER_SCRIPT, screencastSupport, serviceWorkerPolicy, sharedWorkersAllowed, unloadWriteInterception, } from "../browsers.js";
|
|
19
19
|
import { revealedLines } from "./hover.js";
|
|
20
20
|
import { FORMS_INVENTORY_SCRIPT, FORM_PROBE_BODY, formProbeExpression, formIdentity, formStatus, FORMS_READ_FAILED, FORMS_SUBMIT_UNMATCHED, isEmptySubmit, sameControl, isNavigationTeardown, submits, tracksForm, } from "./forms.js";
|
|
21
21
|
import { explainLaunchFailure, isMissingBrowser } from "./launch.js";
|
|
@@ -26,6 +26,7 @@ import { answersWithRefusal, destructiveRefusal, isDestructive, isDestructiveWir
|
|
|
26
26
|
import { scanProject } from "../scan.js";
|
|
27
27
|
import { analyzeDesign, DESIGN_COLLECT_SCRIPT } from "./design.js";
|
|
28
28
|
import { acceptMatches, generatedUpload } from "./fixtures.js";
|
|
29
|
+
import { bodyDigest, headerOf, isReadMethod, judgeUnseenWrite, pausedRequestBytes, RoutedWrites, UnseenRefusals, unseenWriteSource, } from "./unload.js";
|
|
29
30
|
const SETTLE_MS = 400;
|
|
30
31
|
/**
|
|
31
32
|
* Request URL → pathname, falling back to the raw string for anything
|
|
@@ -504,6 +505,19 @@ export class BrowserEngine {
|
|
|
504
505
|
engineName = "chromium";
|
|
505
506
|
/** Non-GET requests fired since the last action — surfaces silent state mutation in read-only runs (timestamped for attribution). */
|
|
506
507
|
mutationRequests = [];
|
|
508
|
+
/**
|
|
509
|
+
* Chromium only (browsers.ts `unloadWriteInterception`): the writes the
|
|
510
|
+
* route handler already let through, so the browser-level interception that
|
|
511
|
+
* catches what it never sees does not judge them twice. That session lives
|
|
512
|
+
* as long as the browser, so a page closed at the end still meets it.
|
|
513
|
+
*/
|
|
514
|
+
routedWrites = new RoutedWrites();
|
|
515
|
+
/** Writes refused at the browser level, so the driver's own report of them (a redirect hop failing) is known as the policy's doing. */
|
|
516
|
+
unseenRefusals = new UnseenRefusals();
|
|
517
|
+
/** Whether the write policy stopped this request, at the route handler or at the browser level. */
|
|
518
|
+
refusedByAnyPolicy(req) {
|
|
519
|
+
return this.refusedByPolicy.has(req) || this.unseenRefusals.has(req.method(), req.url());
|
|
520
|
+
}
|
|
507
521
|
/**
|
|
508
522
|
* Every request answered since the last action, with its status. The HTTP
|
|
509
523
|
* oracle sees each 4xx as it happens and reports it on its own; this ledger
|
|
@@ -580,7 +594,7 @@ export class BrowserEngine {
|
|
|
580
594
|
url: req.url(),
|
|
581
595
|
status,
|
|
582
596
|
resourceType: req.resourceType(),
|
|
583
|
-
blockedByPolicy: this.
|
|
597
|
+
blockedByPolicy: this.refusedByAnyPolicy(req),
|
|
584
598
|
});
|
|
585
599
|
}
|
|
586
600
|
/**
|
|
@@ -700,7 +714,7 @@ export class BrowserEngine {
|
|
|
700
714
|
this.projectDirNote = ` (its real path could not be resolved: ${err instanceof Error ? err.message : String(err)} — a symlinked project path may be wrongly refused)`;
|
|
701
715
|
}
|
|
702
716
|
this.oracles = new OracleMonitor();
|
|
703
|
-
this.oracles.setPolicyRefusalCheck((req) => this.
|
|
717
|
+
this.oracles.setPolicyRefusalCheck((req) => this.refusedByAnyPolicy(req));
|
|
704
718
|
// A request an embed sends to the app is the app's to answer: only one headed outside the app is the embed's.
|
|
705
719
|
this.oracles.setEmbedAttribution((req) => {
|
|
706
720
|
let site = null;
|
|
@@ -934,8 +948,10 @@ export class BrowserEngine {
|
|
|
934
948
|
return refuse(`sent from a page of ${offApp}, outside the app`);
|
|
935
949
|
// Auth/session flows must work in every mode — but never a destructive
|
|
936
950
|
// one, and in observe only the requests a login itself needs.
|
|
937
|
-
if (isAuthExempt(this.mode, method, pathname, destructiveWire))
|
|
951
|
+
if (isAuthExempt(this.mode, method, pathname, destructiveWire)) {
|
|
952
|
+
this.routedWrites.note(this.mode, method, url, bodyDigest(req.postDataBuffer()));
|
|
938
953
|
return route.continue();
|
|
954
|
+
}
|
|
939
955
|
let owned = this.isOwnedResource(pathname);
|
|
940
956
|
// A single UI action commonly fires create-then-immediately-save
|
|
941
957
|
// (POST gets an id, PUT saves content under it) faster than the
|
|
@@ -969,10 +985,22 @@ export class BrowserEngine {
|
|
|
969
985
|
.finally(() => this.pendingCreations.delete(task));
|
|
970
986
|
this.pendingCreations.add(task);
|
|
971
987
|
}
|
|
988
|
+
this.routedWrites.note(this.mode, method, url, bodyDigest(req.postDataBuffer()));
|
|
972
989
|
return route.continue();
|
|
973
990
|
}
|
|
974
991
|
return refuse();
|
|
975
992
|
});
|
|
993
|
+
// Chromium never routes a write a page sends as it is being left; it is judged at the browser level instead.
|
|
994
|
+
if (unloadWriteInterception(this.engineName) === "browser-fetch" && this.browser) {
|
|
995
|
+
try {
|
|
996
|
+
await this.interceptUnseenWrites(this.browser);
|
|
997
|
+
}
|
|
998
|
+
catch (err) {
|
|
999
|
+
// Fail closed: without it, a page being left would write past the policy.
|
|
1000
|
+
await this.close();
|
|
1001
|
+
throw new Error(`Could not start the write policy's browser-level interception: ${err instanceof Error ? err.message.split("\n")[0] : String(err)}`);
|
|
1002
|
+
}
|
|
1003
|
+
}
|
|
976
1004
|
}
|
|
977
1005
|
// Popups / target=_blank: adopt same-origin pages as the active page (with
|
|
978
1006
|
// oracles attached); close foreign-origin popups so exploration cannot
|
|
@@ -999,7 +1027,7 @@ export class BrowserEngine {
|
|
|
999
1027
|
this.lastSnap = null;
|
|
1000
1028
|
}
|
|
1001
1029
|
else {
|
|
1002
|
-
void
|
|
1030
|
+
void BrowserEngine.leaveAndClose(newPage);
|
|
1003
1031
|
}
|
|
1004
1032
|
})
|
|
1005
1033
|
.catch(() => { });
|
|
@@ -1818,6 +1846,121 @@ export class BrowserEngine {
|
|
|
1818
1846
|
return null;
|
|
1819
1847
|
return foreign;
|
|
1820
1848
|
}
|
|
1849
|
+
/**
|
|
1850
|
+
* Pause, at the browser level, every request the context's route handler
|
|
1851
|
+
* lets through or never sees (unload.ts says why), and judge the writes it
|
|
1852
|
+
* never saw. Everything else is sent on at once.
|
|
1853
|
+
*/
|
|
1854
|
+
async interceptUnseenWrites(browser) {
|
|
1855
|
+
this.routedWrites.clear();
|
|
1856
|
+
const session = await browser.newBrowserCDPSession();
|
|
1857
|
+
session.on("Fetch.requestPaused", (event) => void this.judgeUnseenRequest(session, event));
|
|
1858
|
+
await session.send("Fetch.enable", { patterns: [{ urlPattern: "*" }] });
|
|
1859
|
+
}
|
|
1860
|
+
async judgeUnseenRequest(session, event) {
|
|
1861
|
+
const { url, method, headers } = event.request;
|
|
1862
|
+
// Every paused request gets exactly one answer, whatever throws on the way: a request left paused hangs its page.
|
|
1863
|
+
let answered = false;
|
|
1864
|
+
const answer = async (allow) => {
|
|
1865
|
+
answered = true;
|
|
1866
|
+
await (allow
|
|
1867
|
+
? session.send("Fetch.continueRequest", { requestId: event.requestId })
|
|
1868
|
+
: session.send("Fetch.failRequest", { requestId: event.requestId, errorReason: "BlockedByClient" })).catch(() => {
|
|
1869
|
+
/* the request or the browser is already gone */
|
|
1870
|
+
});
|
|
1871
|
+
};
|
|
1872
|
+
try {
|
|
1873
|
+
// Reads, and everything in destructive, go on. A later hop of a redirect is judged like any write: the route handler never sees one.
|
|
1874
|
+
if (isReadMethod(method) || this.mode === "destructive")
|
|
1875
|
+
return await answer(true);
|
|
1876
|
+
const bytes = pausedRequestBytes(event.request);
|
|
1877
|
+
if (this.routedWrites.claim(this.mode, method, url, bodyDigest(bytes)))
|
|
1878
|
+
return await answer(true);
|
|
1879
|
+
let verdict;
|
|
1880
|
+
let pathname = url;
|
|
1881
|
+
try {
|
|
1882
|
+
pathname = pathnameOf(url);
|
|
1883
|
+
const { foreign, offApp } = unseenWriteSource({
|
|
1884
|
+
appUrl: this.baseUrl,
|
|
1885
|
+
mode: this.mode,
|
|
1886
|
+
url,
|
|
1887
|
+
originHeader: headerOf(headers, "origin"),
|
|
1888
|
+
referer: headerOf(headers, "referer"),
|
|
1889
|
+
currentPageUrl: this.page?.url(),
|
|
1890
|
+
trustedEmbeds: this.trustedEmbeds,
|
|
1891
|
+
movedByEmbed: this.embedMoves.movedTo,
|
|
1892
|
+
});
|
|
1893
|
+
verdict = judgeUnseenWrite({
|
|
1894
|
+
mode: this.mode,
|
|
1895
|
+
method,
|
|
1896
|
+
pathname,
|
|
1897
|
+
destructiveWire: isDestructiveWire(pathname, bytes?.toString("utf8")),
|
|
1898
|
+
foreign,
|
|
1899
|
+
offApp,
|
|
1900
|
+
owned: this.isOwnedResource(pathname),
|
|
1901
|
+
});
|
|
1902
|
+
}
|
|
1903
|
+
catch (err) {
|
|
1904
|
+
// Fail closed: a write that cannot be judged is refused, and reported as refused below.
|
|
1905
|
+
console.error(`[scenescout] write policy: could not judge ${method} at the browser level, refused: ${err instanceof Error ? err.message : String(err)}`);
|
|
1906
|
+
verdict = { allow: false };
|
|
1907
|
+
}
|
|
1908
|
+
// Known as refused before the browser fails it, so the driver's report of the failure finds it.
|
|
1909
|
+
if (!verdict.allow)
|
|
1910
|
+
this.unseenRefusals.note(method, url);
|
|
1911
|
+
await answer(verdict.allow);
|
|
1912
|
+
if (verdict.allow) {
|
|
1913
|
+
// Reported like any other write that went out, as the request event does for a routed one.
|
|
1914
|
+
if (!BENIGN_MUTATION_RE.test(url) && this.mutationRequests.length < 20)
|
|
1915
|
+
this.mutationRequests.push({ at: Date.now(), sig: `${method} ${url.slice(0, 120)}` });
|
|
1916
|
+
return;
|
|
1917
|
+
}
|
|
1918
|
+
// Refused and reported as any refusal is. Dropped rather than answered: the page that sent it is going or gone.
|
|
1919
|
+
if (this.blockedRequests.length < 20)
|
|
1920
|
+
this.blockedRequests.push({
|
|
1921
|
+
at: Date.now(),
|
|
1922
|
+
sig: `${method} ${url.slice(0, 140)}`,
|
|
1923
|
+
answered: false,
|
|
1924
|
+
why: verdict.why,
|
|
1925
|
+
type: (event.resourceType ?? "other").toLowerCase(),
|
|
1926
|
+
});
|
|
1927
|
+
this.logAction({ action: "write-policy:blocked", target: `${method} ${pathname}${verdict.why ? ` (${verdict.why})` : ""}`, url: this.page?.url() ?? "" });
|
|
1928
|
+
this.oracles.notePolicyBlock();
|
|
1929
|
+
}
|
|
1930
|
+
catch (err) {
|
|
1931
|
+
console.error(`[scenescout] write policy: browser-level interception failed on ${method}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1932
|
+
}
|
|
1933
|
+
finally {
|
|
1934
|
+
if (!answered) {
|
|
1935
|
+
// Fail closed, and still say so: a refusal nobody hears about reads as a write that went out.
|
|
1936
|
+
try {
|
|
1937
|
+
this.unseenRefusals.note(method, url);
|
|
1938
|
+
}
|
|
1939
|
+
catch {
|
|
1940
|
+
/* the error above is already logged */
|
|
1941
|
+
}
|
|
1942
|
+
await answer(false);
|
|
1943
|
+
try {
|
|
1944
|
+
if (this.blockedRequests.length < 20)
|
|
1945
|
+
this.blockedRequests.push({
|
|
1946
|
+
at: Date.now(),
|
|
1947
|
+
sig: `${method} ${url.slice(0, 140)}`,
|
|
1948
|
+
answered: false,
|
|
1949
|
+
type: (event.resourceType ?? "other").toLowerCase(),
|
|
1950
|
+
});
|
|
1951
|
+
this.logAction({
|
|
1952
|
+
action: "write-policy:blocked",
|
|
1953
|
+
target: `${method} ${url.slice(0, 140)} (not judged: an error in the policy)`,
|
|
1954
|
+
url: this.page?.url() ?? "",
|
|
1955
|
+
});
|
|
1956
|
+
this.oracles.notePolicyBlock();
|
|
1957
|
+
}
|
|
1958
|
+
catch (err) {
|
|
1959
|
+
console.error(`[scenescout] write policy: could not report a refusal: ${err instanceof Error ? err.message : String(err)}`);
|
|
1960
|
+
}
|
|
1961
|
+
}
|
|
1962
|
+
}
|
|
1963
|
+
}
|
|
1821
1964
|
/** Report (and clear) write-policy blocks since the last action. */
|
|
1822
1965
|
drainBlocked() {
|
|
1823
1966
|
this.lastActionBlocked = this.blockedRequests.length;
|
|
@@ -1868,7 +2011,7 @@ export class BrowserEngine {
|
|
|
1868
2011
|
// to see the DUPLICATES that the reporting dedup below intentionally hides.
|
|
1869
2012
|
this.lastActionMutationSigs = this.mutationRequests.map((e) => e.sig);
|
|
1870
2013
|
const fresh = this.mutationRequests
|
|
1871
|
-
.filter((entry) => !this.
|
|
2014
|
+
.filter((entry) => !entry.req || !this.refusedByAnyPolicy(entry.req))
|
|
1872
2015
|
.filter((entry) => {
|
|
1873
2016
|
const key = entry.sig.split("?")[0];
|
|
1874
2017
|
if (this.reportedMutationSigs.has(key))
|
|
@@ -3614,6 +3757,23 @@ export class BrowserEngine {
|
|
|
3614
3757
|
}
|
|
3615
3758
|
}
|
|
3616
3759
|
}
|
|
3760
|
+
/**
|
|
3761
|
+
* Leave a page for about:blank. A write a page sends as it is closed (a
|
|
3762
|
+
* sendBeacon or keepalive fetch on pagehide) is not routed in Firefox or
|
|
3763
|
+
* WebKit, so it reached the server in every mode; one it sends as it
|
|
3764
|
+
* navigates is routed and judged like any other. (Chromium routes neither;
|
|
3765
|
+
* its browser-level interception catches both.)
|
|
3766
|
+
*/
|
|
3767
|
+
static async leave(page) {
|
|
3768
|
+
if (page.isClosed() || page.url() === "about:blank")
|
|
3769
|
+
return;
|
|
3770
|
+
await page.goto("about:blank", { timeout: 3000 }).catch(() => { });
|
|
3771
|
+
}
|
|
3772
|
+
/** Close a page the engine will not drive, after leaving it (`leave`), so what it sends on its way out meets the policy. */
|
|
3773
|
+
static async leaveAndClose(page) {
|
|
3774
|
+
await BrowserEngine.leave(page);
|
|
3775
|
+
await page.close().catch(() => { });
|
|
3776
|
+
}
|
|
3617
3777
|
async close() {
|
|
3618
3778
|
// Marks the end of the time this session held a browser, so the pace
|
|
3619
3779
|
// section can say how long it was held with nothing happening. Only when a
|
|
@@ -3621,6 +3781,10 @@ export class BrowserEngine {
|
|
|
3621
3781
|
// is not an event.
|
|
3622
3782
|
if (this.page)
|
|
3623
3783
|
this.logAction({ action: "close", url: this.page.isClosed() ? "" : this.page.url() });
|
|
3784
|
+
// Every page is left before it is closed, while the write policy still holds, so what a page sends on its way out
|
|
3785
|
+
// is judged and its refusal logged before the flush below.
|
|
3786
|
+
const pages = this.context?.pages() ?? [];
|
|
3787
|
+
await BrowserEngine.settleWithin(Promise.allSettled(pages.map((p) => BrowserEngine.leave(p))), 5000);
|
|
3624
3788
|
// Pending debounced coverage writes must land before the process can exit.
|
|
3625
3789
|
try {
|
|
3626
3790
|
this.memory?.flush();
|
|
@@ -3636,13 +3800,16 @@ export class BrowserEngine {
|
|
|
3636
3800
|
// If teardown overruns the cap, the leftover process is reaped by the
|
|
3637
3801
|
// orphan cleaner on the next attach (or server start).
|
|
3638
3802
|
await BrowserEngine.settleWithin((async () => {
|
|
3639
|
-
|
|
3803
|
+
for (const p of this.context?.pages() ?? [])
|
|
3804
|
+
await p.close().catch(() => { });
|
|
3640
3805
|
await this.context?.close().catch(() => { });
|
|
3641
3806
|
await this.browser?.close().catch(() => { });
|
|
3642
3807
|
})(), 8000);
|
|
3643
3808
|
this.page = null;
|
|
3644
3809
|
this.context = null;
|
|
3645
3810
|
this.browser = null;
|
|
3811
|
+
this.routedWrites.clear();
|
|
3812
|
+
this.unseenRefusals.clear();
|
|
3646
3813
|
this.refs.clear();
|
|
3647
3814
|
this.snapshotUrl = "";
|
|
3648
3815
|
this.currentFingerprint = "";
|
|
@@ -20,7 +20,8 @@
|
|
|
20
20
|
*
|
|
21
21
|
* Pure, so every rule here is table-tested.
|
|
22
22
|
*/
|
|
23
|
-
import {
|
|
23
|
+
import { requestsPair, statedRequests, templatedPathsMatch } from "./lane.js";
|
|
24
|
+
import { failingSignatures, isWorthALook } from "./memory.js";
|
|
24
25
|
/**
|
|
25
26
|
* Upper edge of each confidence bucket. Five is enough to see a shape and few
|
|
26
27
|
* enough that each holds a usable count on a run of a few dozen decisions;
|
|
@@ -86,7 +87,8 @@ function usableConfidence(value) {
|
|
|
86
87
|
*/
|
|
87
88
|
export function calibrate(decisions, findings) {
|
|
88
89
|
const filedKeys = new Map();
|
|
89
|
-
|
|
90
|
+
// Filed as worth a look is not filed as a defect: a lane that called it a defect was not agreed with.
|
|
91
|
+
for (const f of findings.filter((x) => !isWorthALook(x))) {
|
|
90
92
|
if (!f.evidence)
|
|
91
93
|
continue;
|
|
92
94
|
// A finding verified as still present is the most informative match, so it
|
|
@@ -101,8 +103,11 @@ export function calibrate(decisions, findings) {
|
|
|
101
103
|
const claims = decisions.filter((d) => d.verdict === "defect" && d.evidence !== null);
|
|
102
104
|
const checkable = claims.filter((d) => usableConfidence(d.confidence) !== null && joinKeys(d.evidence).size > 0);
|
|
103
105
|
const unjoinable = claims.length - checkable.length;
|
|
106
|
+
const worthALook = decisions.filter((d) => d.verdict === "worth_a_look").length;
|
|
104
107
|
if (checkable.length === 0)
|
|
105
|
-
return unjoinable > 0
|
|
108
|
+
return unjoinable > 0 || worthALook > 0
|
|
109
|
+
? { checkable: 0, filed: 0, stated: 0, buckets: [], ece: 0, verified: { present: 0, gone: 0, changed: 0 }, unjoinable, worthALook }
|
|
110
|
+
: null;
|
|
106
111
|
const buckets = BUCKET_EDGES.map(() => ({ n: 0, conf: 0, hits: 0 }));
|
|
107
112
|
const verified = { present: 0, gone: 0, changed: 0 };
|
|
108
113
|
const matched = new Map();
|
|
@@ -146,7 +151,7 @@ export function calibrate(decisions, findings) {
|
|
|
146
151
|
ece += (b.n / n) * Math.abs(meanConf - rate);
|
|
147
152
|
out.push({ label: bucketLabel(i), decisions: b.n, stated: meanConf, filed: rate });
|
|
148
153
|
});
|
|
149
|
-
return { checkable: n, filed, stated: stated / n, buckets: out, ece, verified, unjoinable };
|
|
154
|
+
return { checkable: n, filed, stated: stated / n, buckets: out, ece, verified, unjoinable, worthALook };
|
|
150
155
|
}
|
|
151
156
|
const pct = (x) => `${Math.round(x * 100)}%`;
|
|
152
157
|
/**
|
|
@@ -162,7 +167,7 @@ export function formatCalibration(c) {
|
|
|
162
167
|
// and a run that recorded none look identical when the section simply
|
|
163
168
|
// vanishes, and the reader concludes the feature is broken — which is this
|
|
164
169
|
// project's own definition of a silent path.
|
|
165
|
-
const had = c.checkable + c.unjoinable;
|
|
170
|
+
const had = c.checkable + c.unjoinable + c.worthALook;
|
|
166
171
|
if (had === 0)
|
|
167
172
|
return [];
|
|
168
173
|
return [
|
|
@@ -170,6 +175,7 @@ export function formatCalibration(c) {
|
|
|
170
175
|
``,
|
|
171
176
|
`Not enough to say yet: ${c.checkable} lane decision(s) could be checked${c.unjoinable > 0 ? ` (and ${c.unjoinable} could not be looked up at all)` : ""}, ` +
|
|
172
177
|
`and ${MIN_FOR_A_VERDICT} are needed before a calibration figure survives one of them changing.`,
|
|
178
|
+
...worthALookNote(c),
|
|
173
179
|
``,
|
|
174
180
|
];
|
|
175
181
|
}
|
|
@@ -193,6 +199,7 @@ export function formatCalibration(c) {
|
|
|
193
199
|
if (c.unjoinable > 0) {
|
|
194
200
|
lines.push(``, `${c.unjoinable} further decision(s) called a defect but named no failing endpoint, so nothing could be looked up for them. They are excluded above rather than counted as wrong.`);
|
|
195
201
|
}
|
|
202
|
+
lines.push(...worthALookNote(c));
|
|
196
203
|
const seen = c.verified.present + c.verified.gone + c.verified.changed;
|
|
197
204
|
if (seen > 0) {
|
|
198
205
|
lines.push(``, `${seen} of the findings those decisions matched ${seen === 1 ? "has" : "have"} since been re-tested with \`scout_verify\`: ${c.verified.present} still present, ${c.verified.gone} gone, ${c.verified.changed} changed. ` +
|
|
@@ -201,6 +208,16 @@ export function formatCalibration(c) {
|
|
|
201
208
|
lines.push(``);
|
|
202
209
|
return lines;
|
|
203
210
|
}
|
|
211
|
+
/** The line saying how many "worth a look" decisions were left out, and why; nothing when there were none. */
|
|
212
|
+
function worthALookNote(c) {
|
|
213
|
+
if (c.worthALook === 0)
|
|
214
|
+
return [];
|
|
215
|
+
return [
|
|
216
|
+
``,
|
|
217
|
+
`${c.worthALook} decision(s) were marked worth a look: real, and a defect only under a convention of the project the run cannot see. ` +
|
|
218
|
+
`Whether one was filed says nothing about whether the lane was right, so they are not scored.`,
|
|
219
|
+
];
|
|
220
|
+
}
|
|
204
221
|
/** Below this, the number swings on a single decision and is worse than no number. */
|
|
205
222
|
export const MIN_FOR_A_VERDICT = 8;
|
|
206
223
|
/**
|
|
@@ -226,45 +243,130 @@ export const MAX_UNFILED_NAMED = 10;
|
|
|
226
243
|
* nobody — and it is cheapest to catch at the moment the planner folds the
|
|
227
244
|
* report, while the lane's session is still open.
|
|
228
245
|
*
|
|
229
|
-
*
|
|
230
|
-
*
|
|
231
|
-
*
|
|
232
|
-
*
|
|
246
|
+
* The check exists to catch unfiled defects, so a false "filed" costs more
|
|
247
|
+
* than a false alarm, which only asks someone to look. A decision counts as
|
|
248
|
+
* filed by one of three rules, tried in order:
|
|
249
|
+
*
|
|
250
|
+
* 1. Failing signatures. When the decision's evidence names one or more
|
|
251
|
+
* failing requests (the store's `failingSignatures`), it is filed only when
|
|
252
|
+
* EVERY one of them is covered by a finding's signature: same method, same
|
|
253
|
+
* status, same route, where a path-template segment (`{id}`, `:id`, `*`) on
|
|
254
|
+
* either side stands for one id segment (see `templatedPathsMatch`). So a
|
|
255
|
+
* finding filed as `GET /api/things/{id} 500` covers `GET /api/things/7 500`,
|
|
256
|
+
* and a decision naming two failing requests is not filed by a finding for
|
|
257
|
+
* one of them.
|
|
258
|
+
* 2. A request that did not fail. When the decision names a request with a
|
|
259
|
+
* 2xx or 3xx status, or none, and a finding names the same request
|
|
260
|
+
* (template-aware, same status), the decision is filed only when its
|
|
261
|
+
* evidence, with its path written as the finding's, is the finding's
|
|
262
|
+
* evidence exactly, restates part of it, or is the finding's evidence
|
|
263
|
+
* followed only by that request's other status (`… 200 as role=viewer
|
|
264
|
+
* (expected 403)` against `… 200 as role=viewer`). Word overlap is not enough
|
|
265
|
+
* there: a 200 names a resource, not a defect, and two different defects
|
|
266
|
+
* on one endpoint read alike. A failing signature in the decision does not
|
|
267
|
+
* block this when it names the same request (`POST /api/things/7/archive
|
|
268
|
+
* 200 (expected 403)` reads as a 403 to the store); one naming any other
|
|
269
|
+
* request does, and sends the decision to rule 1's verdict.
|
|
270
|
+
* 3. Text. Evidence naming no failing request is matched on its text: the
|
|
271
|
+
* same evidence, a restatement of part of it, or shared identifiers and
|
|
272
|
+
* near-identical wording. The calibration join refuses a text match,
|
|
273
|
+
* because there a miss is scored against the lane.
|
|
233
274
|
*/
|
|
234
275
|
export function unfiledDefects(decisions, findings) {
|
|
235
|
-
const keys =
|
|
276
|
+
const keys = [];
|
|
236
277
|
const filed = [];
|
|
237
|
-
|
|
278
|
+
// A defect filed only as worth a look is not in the report's findings, so it is still unfiled.
|
|
279
|
+
for (const f of findings.filter((x) => !isWorthALook(x))) {
|
|
238
280
|
const text = `${f.evidence ?? ""} ${f.title}`;
|
|
281
|
+
const requests = statedRequests(f.evidence ?? "");
|
|
282
|
+
// The store's signatures, and the finding's own failing requests as
|
|
283
|
+
// written: the store reads no `*` in a path, so `GET /api/things/* 500`
|
|
284
|
+
// has no signature of its own.
|
|
239
285
|
if (f.evidence)
|
|
240
286
|
for (const key of joinKeys(f.evidence))
|
|
241
|
-
keys.
|
|
242
|
-
|
|
287
|
+
keys.push(...statedRequests(key));
|
|
288
|
+
keys.push(...requests.filter((q) => q.status !== null && /^[45]/.test(q.status)));
|
|
289
|
+
filed.push({
|
|
290
|
+
ids: identifiers(text),
|
|
291
|
+
words: words(text),
|
|
292
|
+
evidenceWords: words(f.evidence ?? ""),
|
|
293
|
+
evidence: squash(f.evidence ?? ""),
|
|
294
|
+
raw: f.evidence ?? "",
|
|
295
|
+
requests,
|
|
296
|
+
});
|
|
243
297
|
}
|
|
244
298
|
const out = [];
|
|
245
299
|
for (const d of decisions) {
|
|
246
300
|
if (d.verdict !== "defect")
|
|
247
301
|
continue;
|
|
248
302
|
if (d.evidence) {
|
|
249
|
-
const
|
|
250
|
-
|
|
303
|
+
const evidence = d.evidence;
|
|
304
|
+
const joined = [...joinKeys(evidence)];
|
|
305
|
+
const failing = joined.flatMap((sig) => statedRequests(sig));
|
|
306
|
+
if (failing.length > 0 && failing.every((sig) => keys.some((k) => requestsPair(sig, k))))
|
|
307
|
+
continue;
|
|
308
|
+
if (filed.some((f) => restatesRequest(evidence, failing, f)))
|
|
251
309
|
continue;
|
|
252
310
|
// A failing-endpoint signature is the store's own identity for a bug. When
|
|
253
|
-
// the decision has one and no finding
|
|
311
|
+
// the decision has one and no finding covers it, nothing else is a match.
|
|
254
312
|
if (joined.length > 0) {
|
|
255
313
|
out.push(`${d.observation} — ${d.evidence}`);
|
|
256
314
|
continue;
|
|
257
315
|
}
|
|
258
|
-
|
|
259
|
-
const w = words(d.evidence);
|
|
260
|
-
const text = squash(d.evidence);
|
|
261
|
-
if (filed.some((f) => (text !== "" && f.evidence === text) || restates(text, f) || covers(ids, w, f)))
|
|
316
|
+
if (filed.some((f) => matchesText(evidence, f)))
|
|
262
317
|
continue;
|
|
263
318
|
}
|
|
264
319
|
out.push(d.evidence ? `${d.observation} — ${d.evidence}` : d.observation);
|
|
265
320
|
}
|
|
266
321
|
return out;
|
|
267
322
|
}
|
|
323
|
+
/** Rule 3: the same evidence, a restatement of part of it, or shared identifiers and wording. */
|
|
324
|
+
function matchesText(reported, f) {
|
|
325
|
+
const text = squash(reported);
|
|
326
|
+
return (text !== "" && f.evidence === text) || restates(text, f) || covers(identifiers(reported), words(reported), f);
|
|
327
|
+
}
|
|
328
|
+
/**
|
|
329
|
+
* Rule 2 of unfiledDefects: the decision names a request that did not fail,
|
|
330
|
+
* the finding names the same request, and the decision's evidence — its path
|
|
331
|
+
* written as the finding's, so a template and the id it stands for are not a
|
|
332
|
+
* difference — is the finding's evidence exactly, restates part of it, or
|
|
333
|
+
* adds only the status the request should have returned.
|
|
334
|
+
* `failing` is the decision's failing signatures; each must name this same
|
|
335
|
+
* request, or the pair is refused.
|
|
336
|
+
*/
|
|
337
|
+
function restatesRequest(evidence, failing, f) {
|
|
338
|
+
for (const r of statedRequests(evidence)) {
|
|
339
|
+
if (r.status !== null && /^[45]/.test(r.status))
|
|
340
|
+
continue;
|
|
341
|
+
const paired = f.requests.find((q) => requestsPair(r, q));
|
|
342
|
+
if (!paired)
|
|
343
|
+
continue;
|
|
344
|
+
if (!failing.every((sig) => sig.method === r.method && templatedPathsMatch(sig.segments, r.segments)))
|
|
345
|
+
continue;
|
|
346
|
+
const aligned = squash(evidence.slice(0, r.start) + f.raw.slice(paired.start, paired.end) + evidence.slice(r.end));
|
|
347
|
+
if (aligned !== "" && (f.evidence === aligned || restates(aligned, f) || addsOnlyExpectedStatus(aligned, f.evidence, r.status)))
|
|
348
|
+
return true;
|
|
349
|
+
}
|
|
350
|
+
return false;
|
|
351
|
+
}
|
|
352
|
+
/**
|
|
353
|
+
* The status a lane adds after the call it quotes, to say what the call
|
|
354
|
+
* should have returned: "(expected 403)", "(reject 403)", "reject 403". One
|
|
355
|
+
* status, an optional verb, optional brackets, and nothing else.
|
|
356
|
+
*/
|
|
357
|
+
const EXPECTED_STATUS_RE = /^[\s,;]*\(?\s*(?:(?:expected|expect|rejected|reject)\s*:?\s*)?([1-5]\d{2})\s*\)?[\s.,;]*$/;
|
|
358
|
+
/**
|
|
359
|
+
* Whether the reported evidence is the filed evidence followed by nothing but
|
|
360
|
+
* the status the request should have returned, which differs from the one it
|
|
361
|
+
* did. Anything else after it — another request, another role, another
|
|
362
|
+
* claim — is something the finding does not say, so it is not covered.
|
|
363
|
+
*/
|
|
364
|
+
function addsOnlyExpectedStatus(reported, filed, status) {
|
|
365
|
+
if (filed === "" || !reported.startsWith(filed))
|
|
366
|
+
return false;
|
|
367
|
+
const expected = EXPECTED_STATUS_RE.exec(reported.slice(filed.length))?.[1];
|
|
368
|
+
return expected !== undefined && expected !== status;
|
|
369
|
+
}
|
|
268
370
|
/**
|
|
269
371
|
* Shortest reported evidence that counts as restating a filing it is part of.
|
|
270
372
|
*/
|