scenescout 3.6.0 → 3.6.1

package/CHANGELOG.md

@@ -1,5 +1,13 @@
 # scenescout
 
+## 3.6.1
+
+### Patch Changes
+
+- 90d1aec: Two findings that share only a quoted string are merged only when their kinds are one family (data, failures, presentation, flow, security, performance, and "other" on its own). A layout defect naming the button it covers ("Save notes") was being folded into the data defect about what that button does, and dropped from the report. The same bug filed twice under neighbouring categories — data-loss and data-inconsistency, page-error and console-error — still merges.
+- 8abf639: The false-success check recognises a refusal the page announces in its own words — "Only an open order can be sent for approval", "You can't delete an approved order", "This order is already approved" — in a status region, alert or dialog, as the page admitting the refusal, instead of reporting the word "sent" or "approved" as a false claim of success. The same wording as ordinary help text elsewhere on the page ("This cannot be undone", "Password must be at least 8 characters") excuses nothing.
+- ca52079: The lane-report fold's check for defects judged but never filed now recognises a filed finding when the lane reworded its evidence, matching on the identifiers that survive rewording — test ids and contrast ratios, two of them in common — or on near-identical wording. Replayed against three archived benchmark runs it raised 6 flags where it had raised 39, and still named every defect that was genuinely left unfiled. One shared test id is never enough, and API paths are not identifiers, so a missed defect on the same button or endpoint as a filed one is still named.
+
 ## 3.6.0
 
 ### Minor Changes

package/dist/engine/calibration.js

@@ -233,13 +233,13 @@ export const MAX_UNFILED_NAMED = 10;
  */
 export function unfiledDefects(decisions, findings) {
     const keys = new Set();
-    const texts = [];
+    const filed = [];
     for (const f of findings) {
-        if (!f.evidence)
-            continue;
-        for (const key of joinKeys(f.evidence))
-            keys.add(key);
-        texts.push(squash(f.evidence));
+        const text = `${f.evidence ?? ""} ${f.title}`;
+        if (f.evidence)
+            for (const key of joinKeys(f.evidence))
+                keys.add(key);
+        filed.push({ ids: identifiers(text), words: words(text), evidenceWords: words(f.evidence ?? ""), evidence: squash(f.evidence ?? "") });
     }
     const out = [];
     for (const d of decisions) {
@@ -250,25 +250,84 @@ export function unfiledDefects(decisions, findings) {
             if (joined.some((k) => keys.has(k)))
                 continue;
             // A failing-endpoint signature is the store's own identity for a bug. When
-            // the decision has one and no finding shares it, text overlap is not a
-            // match: "GET /api/items" inside "... GET /api/items returned 500" is.
+            // the decision has one and no finding shares it, nothing else is a match.
             if (joined.length > 0) {
                 out.push(`${d.observation} — ${d.evidence}`);
                 continue;
             }
+            const ids = identifiers(d.evidence);
+            const w = words(d.evidence);
             const text = squash(d.evidence);
-            if (text && texts.some((t) => t === text || contains(t, text) || contains(text, t)))
+            if (filed.some((f) => (text !== "" && f.evidence === text) || covers(ids, w, f)))
                 continue;
         }
         out.push(d.evidence ? `${d.observation} — ${d.evidence}` : d.observation);
     }
     return out;
 }
-/** One signature inside another, when the inner one is long enough to mean something: "404" is inside half the findings in a run. */
-function contains(outer, inner) {
-    return inner.length >= MIN_CONTAINED && outer.includes(inner);
+function covers(ids, w, f) {
+    let shared = 0;
+    for (const id of ids)
+        if (f.ids.has(id))
+            shared += 1;
+    // Against the evidence alone as well: a finding's title adds words the
+    // lane's report never repeats, and diluted an otherwise identical signature.
+    const overlap = Math.max(jaccard(w, f.words), jaccard(w, f.evidenceWords));
+    return shared >= 2 || overlap >= 0.6;
+}
+/**
+ * Test ids, kebab-case identifiers of three or more parts, and contrast ratios.
+ * Not API paths: a path that answered 2xx names a resource, not a defect — two
+ * different bugs on POST /api/orders share it — and a failing path already has
+ * its own identity in joinKeys.
+ */
+function identifiers(text) {
+    const out = new Set();
+    const t = text.toLowerCase();
+    for (const m of t.matchAll(/testid=["']?([a-z0-9_-]+)/g))
+        out.add(m[1]);
+    for (const m of t.matchAll(/\b[a-z][a-z0-9]*(?:-[a-z0-9]+){2,}\b/g))
+        out.add(m[0]);
+    for (const m of t.matchAll(/\b\d+(?:\.\d+)?:1\b/g))
+        out.add(m[0]);
+    return out;
+}
+const STOP = new Set([
+    "the",
+    "and",
+    "with",
+    "for",
+    "but",
+    "not",
+    "was",
+    "are",
+    "has",
+    "this",
+    "that",
+    "from",
+    "into",
+    "when",
+    "then",
+    "than",
+    "only",
+    "also",
+    "its",
+]);
+function words(text) {
+    return new Set(text
+        .toLowerCase()
+        .split(/[^a-z0-9]+/)
+        .filter((w) => w.length >= 3 && !STOP.has(w)));
+}
+function jaccard(a, b) {
+    if (a.size === 0 || b.size === 0)
+        return 0;
+    let both = 0;
+    for (const x of a)
+        if (b.has(x))
+            both += 1;
+    return both / (a.size + b.size - both);
 }
-const MIN_CONTAINED = 12;
 function squash(text) {
     return text.toLowerCase().replace(/\s+/g, " ").trim();
 }

package/dist/engine/claims.js

@@ -64,11 +64,33 @@ const EMPTY_RE = /\b(?:no (?:results?|items?|records?|rows?|data|entries|matches
  */
 const SUCCESS_RE = /\b(?:success(?:fully)?|saved|created|updated|deleted|removed|submitted|sent|published|approved|completed|added|changes? saved|done)\b/i;
 /**
- * Phrases that admit something went wrong. Their presence is what makes a page
+ * Phrases that admit something went wrong — including a refusal explained in
+ * the app's own words ("Only an open order can be sent for approval", "You
+ * can't delete an approved order"), which is correct behaviour even though it
+ * uses none of the words for an error and may contain one for success. Their presence is what makes a page
  * INNOCENT: an app that refuses a request and says so has behaved correctly,
  * whatever else is on the screen, and must not be reported.
  */
-const ERROR_RE = /\b(?:error|failed|failure|could ?n[o']?t|unable to|went wrong|try again|retry|denied|forbidden|unauthori[sz]ed|not allowed|no permission|timed out|unavailable|problem loading)\b/i;
+const ERROR_RE = /\b(?:error|failed|failure|could ?n[o'’]?t|unable to|went wrong|try again|retry|denied|forbidden|unauthori[sz]ed|not allowed|no permission|timed out|unavailable|problem loading)\b/i;
+/**
+ * A refusal explained in the app's own words: "Only an open order can be sent
+ * for approval", "You can't delete an approved order", "Nothing was saved".
+ * Correct behaviour after a refused request, though it uses none of the words
+ * for an error and may contain one for success ("sent").
+ *
+ * Unlike ERROR_RE, this counts only in text the page ANNOUNCES — a live region
+ * or an alert (see PageState.announced). The same phrasing is ordinary
+ * help text everywhere else ("This cannot be undone", "Password must be at
+ * least 8 characters", "Only admins can invite members"), and page-wide it
+ * excused real lies: a refused delete reporting "Workspace deleted." went
+ * unreported because the Danger zone said "This cannot be undone."
+ */
+const REFUSAL_RE = /\b(?:can(?:not|[’'`]?t| not)|must be|(?:is|are) already|only (?:an? |the )?\w+(?: \w+)? (?:can|may|be)|may only|can only(?: be)?|(?:nothing|not) (?:was|has been|have been|been) (?:saved|sent|deleted|updated|created|changed|submitted)|(?:was|were|has|have|is|are)(?: not|n[’']t)(?: been)? (?:saved|sent|deleted|updated|created|changed|submitted))\b/i;
+/** Whether a piece of announced text explains a refusal. */
+export function isRefusalNotice(text) {
+    const trimmed = text.trim();
+    return trimmed.length > 0 && trimmed.length <= CLAIM_TEXT_MAX && REFUSAL_RE.test(trimmed);
+}
 /** Longest piece of text judged. An empty state is a sentence; a paragraph that happens to contain one of these words is not a claim. */
 export const CLAIM_TEXT_MAX = 120;
 /**
@@ -122,6 +144,10 @@ export function findContradictions(requests, page) {
     // an error banner and a stale empty state is not reported.
     if (claims.includes("error"))
         return [];
+    // A refusal explained in the app's own words, where the page announces its
+    // responses. Help text with the same wording elsewhere excuses nothing.
+    if ((page.announced ?? []).some(isRefusalNotice))
+        return [];
     const out = [];
     const reads = refused.filter((r) => !WRITING_METHODS.has(r.method.toUpperCase()));
     const saysEmpty = claims.includes("empty") || page.emptyLists > 0;
@@ -148,6 +174,8 @@ export function findContradictions(requests, page) {
     }
     return out;
 }
+/** Most announced regions read from one page. */
+export const MAX_ANNOUNCED = 40;
 /** Most pieces of text read from one page. A page with more than this has nothing useful to say in the extra ones. */
 export const MAX_CLAIM_TEXTS = 120;
 /**
@@ -167,6 +195,11 @@ export const MAX_CLAIM_TEXTS = 120;
 export const CLAIM_SCAN_SCRIPT = `(() => {
   const visible = ${VISIBLE_SRC};
   const texts = [];
+  const announced = [];
+  // Where a page SAYS something in response to an action. Not dialogs: a modal
+  // is a container of static text (its form's help, its warning), and counting
+  // it brought back the help text this set exists to exclude.
+  const ANNOUNCES = "[role~='status'], [role~='alert'], [aria-live]:not([aria-live='off']), output";
   const seen = new Set();
   const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_ELEMENT);
   let el = document.body;
@@ -197,5 +230,20 @@ export const CLAIM_SCAN_SCRIPT = `(() => {
     const rows = list.querySelectorAll(":scope > li, :scope > [role='row'], :scope > [role='listitem']");
     if (rows.length === 0) emptyLists += 1;
   }
-  return { texts, emptyLists };
+  // Read on its own, past the cap on page texts: a toast rendered at the end of
+  // a long page is exactly the text that must not be missed.
+  for (const region of document.querySelectorAll(ANNOUNCES)) {
+    if (!visible(region)) continue;
+    // innerText, not textContent: hidden parts of a region are not said. Split
+    // into sentences so a long region is read rather than dropped whole.
+    const said = (region.innerText || "").replace(/\\s+/g, " ").trim();
+    for (const sentence of said.split(/(?<=[.!?])\\s+/)) {
+      if (announced.length >= ${MAX_ANNOUNCED}) break;
+      const s = sentence.trim();
+      if (s && s.length <= ${CLAIM_TEXT_MAX} && !announced.includes(s)) announced.push(s);
+    }
+  }
+
+
+  return { texts, announced, emptyLists };
 })()`;

package/dist/engine/memory.js

@@ -412,6 +412,46 @@ function sameEndpointBug(existing, incoming) {
         return false;
     return sharesEndpointSignature(existing, incoming);
 }
+/**
+ * Families of finding kinds that one bug is plausibly filed under by two
+ * sessions: a crash is a page-error to one and a console-error to another, a
+ * refused save is data-loss to one and data-inconsistency to another. Across
+ * families — a layout defect and a data defect — a shared quoted string names
+ * a place on the page, not a bug.
+ */
+const CATEGORY_FAMILY = {
+    "data-inconsistency": "data",
+    "data-loss": "data",
+    "stale-state": "data",
+    "http-error": "failure",
+    network: "failure",
+    "console-error": "failure",
+    "page-error": "failure",
+    visual: "presentation",
+    "ux-polish": "presentation",
+    a11y: "presentation",
+    "missing-testid": "presentation",
+    "ux-confusing": "flow",
+    "dead-end": "flow",
+    security: "security",
+    "permission-leak": "security",
+    performance: "performance",
+};
+/**
+ * Whether two categories are one family. "other" — the category for nothing
+ * that fits — is a family of its own: as a wildcard, one "other" finding
+ * quoting a control absorbed both the layout and the data finding about it. A
+ * missing or unknown category matches nothing, so a finding read from an older
+ * file never merges on a guess.
+ */
+export function sameFamily(a, b) {
+    if (!a || !b)
+        return false;
+    if (a === "other" || b === "other")
+        return a === b;
+    const fa = CATEGORY_FAMILY[a];
+    return fa !== undefined && fa === CATEGORY_FAMILY[b];
+}
 function sameFinding(a, b) {
     const aEv = a.evidence?.toLowerCase().replace(/\s+/g, " ").trim();
     const bEv = b.evidence?.toLowerCase().replace(/\s+/g, " ").trim();
@@ -426,17 +466,27 @@ function sameFinding(a, b) {
     // them, and collapsed into one. Matching a title literal against the other
     // finding's full text keeps the intended case (one states the string in its
     // title, the other mentions it in its detail).
-    const aTitleLits = findingLiterals(a.title);
-    const bTitleLits = findingLiterals(b.title);
-    if (aTitleLits.size > 0 || bTitleLits.size > 0) {
-        const aAll = findingLiterals(a.title, a.detail, a.evidence);
-        const bAll = findingLiterals(b.title, b.detail, b.evidence);
-        for (const lit of aTitleLits)
-            if (bAll.has(lit))
-                return true;
-        for (const lit of bTitleLits)
-            if (aAll.has(lit))
-                return true;
+    //
+    // Only between findings of one FAMILY of kinds (see sameFamily). A quoted
+    // string is as often the name of a control as a message the app showed, and
+    // a layout defect naming the button it covers ("Save notes") shares that
+    // literal with the data defect describing what the button does — in its
+    // detail or its own title. Merged, the layout defect was filed and then lost
+    // from the report on most runs of a benchmark. Within a family, one bug
+    // filed twice under neighbouring categories still merges.
+    if (sameFamily(a.category, b.category)) {
+        const aTitleLits = findingLiterals(a.title);
+        const bTitleLits = findingLiterals(b.title);
+        if (aTitleLits.size > 0 || bTitleLits.size > 0) {
+            const aAll = findingLiterals(a.title, a.detail, a.evidence);
+            const bAll = findingLiterals(b.title, b.detail, b.evidence);
+            for (const lit of aTitleLits)
+                if (bAll.has(lit))
+                    return true;
+            for (const lit of bTitleLits)
+                if (aAll.has(lit))
+                    return true;
+        }
     }
     // Both carry evidence and neither matched above: distinct bugs, however
     // similar the titles — never fuzzy-merge across differing evidence.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scenescout",
-  "version": "3.6.0",
+  "version": "3.6.1",
   "description": "SceneScout — exploratory UI testing for AI coding agents. An MCP server that gives any agent (Claude Code, Cursor, VS Code Copilot, Codex, Gemini CLI and others) a structured view of a running web app, always-on oracles, a network-level write policy, memory across runs and a gap-checked report.",
   "license": "MIT",
   "author": "brunoboto96",