scenescout 3.14.1 → 3.15.0

package/dist/engine/refresh.js

@@ -12,7 +12,17 @@
  * has already rotated the token, it loads the rotated profile and sends the
  * current token in place of the spent one. When the response arrives and the
  * page has stored what it got back, the session writes its state over the
- * profile and releases the lock. No two sessions ever present one token.
+ * profile and releases the lock, so two sessions do not present one token.
+ *
+ * Which requests count is decided by brokerDecision. A token in the body, the
+ * URL or a header the page set makes the request a refresh. A token that is
+ * only in the Cookie header does not by itself: a refresh cookie scoped to "/"
+ * rides on every request the page makes, scripts and images included. Such a
+ * request counts only when it is plausibly the refresh call, a POST to a path
+ * named for one or an endpoint the broker has seen rotate the cookie. Static
+ * assets never count, and a broker that cannot do its job lets the request
+ * through as the page sent it: an app left without its scripts is worse than
+ * the rare double refresh.
  *
  * Sessions may be separate processes (one MCP server per lane) sharing the
  * profile on disk, so the lock is a file, created exclusively, owner-only,
@@ -37,8 +47,14 @@ export const REFRESH_NAME_RE = /refresh/i;
 export const MIN_TOKEN_LENGTH = 16;
 /** How deep inside a JSON-valued storage entry a refresh-token field is looked for. */
 const MAX_JSON_DEPTH = 4;
+/**
+ * Whether a value could be a token: long enough, no whitespace, and not an
+ * address. An app that keeps the URL it refreshes at under a refresh-named key
+ * ("/auth/refresh", "https://…/token") stores a path, which every request to
+ * that path would otherwise seem to carry.
+ */
 function looksLikeToken(value) {
-    return typeof value === "string" && value.length >= MIN_TOKEN_LENGTH && !/\s/.test(value);
+    return typeof value === "string" && value.length >= MIN_TOKEN_LENGTH && !/\s/.test(value) && !/^(\/|[a-z][a-z0-9+.-]*:\/\/)/i.test(value);
 }
 /** Refresh-token fields inside a JSON value, by dotted path. */
 function jsonFields(value, at, depth, out) {
@@ -66,7 +82,7 @@ export function refreshTokenSlots(state) {
     if (Array.isArray(s.cookies)) {
         for (const c of s.cookies) {
             if (typeof c?.name === "string" && REFRESH_NAME_RE.test(c.name) && looksLikeToken(c.value)) {
-                out.push({ slot: `cookie ${c.name} (${String(c.domain ?? "")}${String(c.path ?? "")})`, value: c.value });
+                out.push({ slot: `cookie ${c.name} (${String(c.domain ?? "")}${String(c.path ?? "")})`, value: c.value, cookie: c.name });
             }
         }
     }
@@ -106,23 +122,142 @@ function wireForms(value) {
     return encoded === value ? [value] : [value, encoded];
 }
 /**
- * The known refresh token a request carries — in its body, a header (a Cookie
- * header included) or its URL — or null. `known` is what the session last
- * loaded from the profile; a token the page got some other way is not the
- * role's and is left alone.
+ * Headers the browser adds to a request by itself. A token found in one of
+ * these was not sent by the page as an act of refreshing: a cookie scoped to
+ * "/" rides on every request the page makes, and a page whose own address
+ * carries a token passes it on as the Referer of everything it loads.
+ */
+function browserAddedHeader(name) {
+    const n = name.toLowerCase();
+    return n === "cookie" || n === "referer" || n === "origin" || n === "host" || n.startsWith("sec-") || n.startsWith(":");
+}
+/**
+ * The known refresh token a request carries, and where, or null. `known` is
+ * what the session last loaded from the profile; a token the page got some
+ * other way is not the role's and is left alone. A token in the body, the URL
+ * or a header the page set wins over one that is only in the Cookie header.
  */
-export function presentedToken(req, known) {
+export function tokenPresence(req, known) {
     if (known.length === 0)
         return null;
-    const haystacks = [req.body ?? "", req.url, ...Object.values(req.headers)];
+    const carries = (haystack, t) => wireForms(t.value).some((form) => haystack.includes(form));
+    const explicit = Object.entries(req.headers)
+        .filter(([name]) => !browserAddedHeader(name))
+        .map(([, value]) => value);
     for (const t of known) {
-        for (const form of wireForms(t.value)) {
-            if (haystacks.some((h) => h.includes(form)))
-                return t;
-        }
+        if (req.body && carries(req.body, t))
+            return { slot: t, via: "body" };
+        if (carries(req.url, t))
+            return { slot: t, via: "url" };
+        if (explicit.some((h) => carries(h, t)))
+            return { slot: t, via: "header" };
     }
+    const cookie = Object.entries(req.headers).find(([name]) => name.toLowerCase() === "cookie")?.[1] ?? "";
+    for (const t of known)
+        if (cookie && carries(cookie, t))
+            return { slot: t, via: "cookie" };
     return null;
 }
+/**
+ * Requests for a page's static assets. They are never brokered, waited on or
+ * dropped, whatever cookie rides on them: holding a script back behind the
+ * refresh lock leaves the app under test without its code.
+ */
+export const STATIC_RESOURCE_TYPES = new Set(["script", "stylesheet", "image", "font", "media", "manifest", "texttrack"]);
+export function isStaticAsset(method, resourceType) {
+    return (method === "GET" || method === "HEAD") && STATIC_RESOURCE_TYPES.has(resourceType);
+}
+/**
+ * Path segments that name a token refresh. Whole segments, as the write
+ * policy's auth segments are, never substrings: `/api/tokens` mints an API
+ * token and `/refreshments` is a menu.
+ */
+const REFRESH_SEGMENT_RE = /^(refresh|refresh[-_]?token|token|renew|reauth|reauthenticate)$/i;
+/** A path whose last one or two segments name a token refresh: /auth/refresh, /oauth/token, /api/token/refresh. */
+export function refreshLikePath(pathname) {
+    const segments = pathname.split("/").filter(Boolean);
+    return segments.slice(-2).some((seg) => REFRESH_SEGMENT_RE.test(seg));
+}
+/** An endpoint as the broker remembers it: the method and the path, without the query. */
+export function endpointKey(method, url) {
+    let pathname;
+    try {
+        pathname = new URL(url).pathname;
+    }
+    catch {
+        pathname = url.split("?")[0];
+    }
+    return `${method.toUpperCase()} ${pathname}`;
+}
+/**
+ * Path segments of a sign-in, a sign-up or a sign-out. A cookie such a request
+ * sets is a new sign-in, perhaps as somebody else, not a refresh: the broker
+ * neither learns the endpoint nor saves the cookie over the role's profile.
+ */
+const SIGN_IN_SEGMENT_RE = /^(login|log-in|signin|sign-in|logout|log-out|signout|sign-out|signup|sign-up|register|verify|otp|magic-link|callback|authorize|sso)$/i;
+/** Whether an endpoint seen to rotate the token may be learned as the refresh call: any but a sign-in, sign-up or sign-out. */
+export function learnableEndpoint(url) {
+    let pathname;
+    try {
+        pathname = new URL(url).pathname;
+    }
+    catch {
+        return false;
+    }
+    return !pathname
+        .split("/")
+        .filter(Boolean)
+        .slice(-2)
+        .some((seg) => SIGN_IN_SEGMENT_RE.test(seg));
+}
+/** A body that asks for a grant other than a refresh (a password or code sign-in) is not a refresh, whatever cookie rides on it. */
+function otherGrant(body) {
+    const m = /grant_type["']?\s*[=:]\s*["']?([A-Za-z_:.-]+)/.exec(body ?? "");
+    return m !== null && !/refresh/i.test(m[1]);
+}
+/**
+ * Whether a token that is only in the Cookie header could make this request a
+ * refresh, so that header is worth reading at all. A cookie alone never does,
+ * since a cookie scoped to "/" rides on every request. Only a request that is
+ * plausibly the refresh call counts: one the broker has learned rotates the
+ * token (`learned`, by endpointKey), or a POST, PUT or PATCH to a path whose
+ * last one or two segments name a refresh.
+ */
+export function cookieMayCount(method, url, learned) {
+    if (learned.has(endpointKey(method, url)))
+        return true;
+    if (method !== "POST" && method !== "PUT" && method !== "PATCH")
+        return false;
+    let pathname;
+    try {
+        pathname = new URL(url).pathname;
+    }
+    catch {
+        return false;
+    }
+    return refreshLikePath(pathname);
+}
+/**
+ * Whether a request goes through the refresh lock. A static asset never does.
+ * A request carrying a known token in its body, URL or a header the page set
+ * does. A token that is only in the Cookie header counts only for a request
+ * that is plausibly the refresh call (cookieMayCount), and not for one whose
+ * body asks for another grant.
+ */
+export function brokerDecision(req, known, learned) {
+    if (isStaticAsset(req.method, req.resourceType))
+        return { kind: "pass", why: "static asset" };
+    const found = tokenPresence(req, known);
+    if (!found)
+        return { kind: "pass", why: "no known token" };
+    if (found.via !== "cookie")
+        return { kind: "broker", sent: found.slot, via: found.via };
+    if (!cookieMayCount(req.method, req.url, learned))
+        return { kind: "pass", why: "only in a cookie" };
+    if (otherGrant(req.body))
+        return { kind: "pass", why: "another grant" };
+    return { kind: "broker", sent: found.slot, via: "cookie" };
+}
 export function planRefresh(sent, current) {
     if (current.some((c) => c.value === sent.value))
         return { kind: "send" };
@@ -180,6 +315,35 @@ export function profileAfterRotation(pageState, profileOnDisk) {
         return pageState;
     return withSessionStorage(pageState, splitProfile(profileOnDisk).sessionStorage);
 }
+/**
+ * The cookies a response's Set-Cookie headers set, by name. A header value may
+ * hold several cookies joined by newlines, as some browsers report them.
+ */
+function setCookiePairs(setCookies) {
+    const out = new Map();
+    for (const line of setCookies.flatMap((l) => l.split("\n"))) {
+        const pair = line.split(";")[0];
+        const eq = pair.indexOf("=");
+        if (eq <= 0)
+            continue;
+        out.set(pair.slice(0, eq).trim(), pair.slice(eq + 1).trim());
+    }
+    return out;
+}
+/**
+ * The known cookie tokens a response's Set-Cookie headers replace with a new
+ * token: the request that got this response refreshed them. A cookie set to an
+ * empty or short value is being cleared (a sign-out), not rotated.
+ */
+export function rotatedCookies(setCookies, known) {
+    const set = setCookiePairs(setCookies);
+    return known.filter((t) => {
+        if (t.cookie === undefined)
+            return false;
+        const next = set.get(t.cookie);
+        return looksLikeToken(next) && next !== t.value;
+    });
+}
 /**
  * The rotated refresh token a refresh response carries, for a page that did
  * not store it in time: a refresh-named field in a JSON body, else a
@@ -198,16 +362,9 @@ export function rotatedFromResponse(body, setCookies) {
         // Not a JSON body: the Set-Cookie headers are all there is to read.
     }
     if (found.size === 0) {
-        for (const line of setCookies) {
-            const pair = line.split(";")[0];
-            const eq = pair.indexOf("=");
-            if (eq <= 0)
-                continue;
-            const name = pair.slice(0, eq).trim();
-            const value = pair.slice(eq + 1).trim();
+        for (const [name, value] of setCookiePairs(setCookies))
             if (REFRESH_NAME_RE.test(name) && looksLikeToken(value))
                 found.add(value);
-        }
     }
     return found.size === 1 ? [...found][0] : null;
 }
@@ -224,6 +381,26 @@ export function swapProfileToken(state, from, to) {
     }
     return s;
 }
+/**
+ * The headers the broker sends a request with when it has to send the request
+ * itself, which it does only to put the current token in a Cookie header (a
+ * browser keeps the Cookie header it built, whatever a route override says).
+ * The page's own headers are kept; the ones the HTTP client sets for itself
+ * (host, content length, connection, the encodings it can decode) and
+ * pseudo-headers are left out, and `cookie` is the line to send.
+ */
+export function headersForResend(headers, cookie) {
+    const out = {};
+    for (const [name, value] of Object.entries(headers)) {
+        const n = name.toLowerCase();
+        if (n.startsWith(":") || ["host", "content-length", "connection", "transfer-encoding", "accept-encoding", "cookie"].includes(n))
+            continue;
+        out[n] = value;
+    }
+    if (cookie)
+        out.cookie = cookie;
+    return out;
+}
 // ── Whether the broker runs ──────────────────────────────────────────────────
 export const REFRESH_BROKER_ENV = "SCENESCOUT_REFRESH_BROKER";
 /**
@@ -255,6 +432,60 @@ export const DEFAULT_WAIT_MS = 45_000;
 export function lockPathFor(profileFile) {
     return `${profileFile}.lock`;
 }
+// ── Endpoints learned to rotate the token ──────────────────────────────────
+/**
+ * Where the broker keeps the endpoints it has seen rotate the role's refresh
+ * token, beside the profile: every session of the role reads it, so a session
+ * brokers an endpoint another session learned, even in another process.
+ */
+export function endpointsPathFor(profileFile) {
+    return `${profileFile}.endpoints`;
+}
+/** At most this many learned endpoints are kept, newest first. */
+export const MAX_LEARNED_ENDPOINTS = 20;
+const ENDPOINT_KEY_RE = /^(GET|POST|PUT|PATCH) \/\S{0,500}$/;
+/** The learned endpoints on disk. None when there is no file yet; a file that is not the broker's teaches nothing. */
+export function readLearnedEndpoints(file) {
+    let raw;
+    try {
+        raw = fs.readFileSync(file, "utf8");
+    }
+    catch (err) {
+        if (err.code === "ENOENT")
+            return [];
+        throw err;
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch {
+        return []; // written by rename, so never half-written: whatever this is, it is not a list of endpoints
+    }
+    const list = parsed?.endpoints;
+    if (!Array.isArray(list))
+        return [];
+    return list.filter((e) => typeof e === "string" && ENDPOINT_KEY_RE.test(e)).slice(0, MAX_LEARNED_ENDPOINTS);
+}
+/** The list with `key` added first, once, and the oldest dropped past the cap. A key that is not an endpoint changes nothing. */
+export function withLearnedEndpoint(list, key) {
+    if (!ENDPOINT_KEY_RE.test(key))
+        return [...list];
+    return [key, ...list.filter((e) => e !== key)].slice(0, MAX_LEARNED_ENDPOINTS);
+}
+/** Write the learned endpoints owner-only, by rename, so a reader never sees half a file. Call it holding the lock. */
+export function writeLearnedEndpoints(file, list) {
+    const temp = `${file}.${process.pid}.tmp`;
+    fs.rmSync(temp, { force: true });
+    const fd = fs.openSync(temp, "wx", LOCK_FILE_MODE);
+    try {
+        fs.writeFileSync(fd, JSON.stringify({ endpoints: list }));
+    }
+    finally {
+        fs.closeSync(fd);
+    }
+    fs.renameSync(temp, file);
+}
 /** Whether a lock last touched at `mtimeMs` is stale at `now`. */
 export function isStale(mtimeMs, now, staleMs) {
     return now - mtimeMs > staleMs;
@@ -275,6 +506,20 @@ function readNonce(file) {
     }
 }
 /** One attempt: the lock, or null when another process holds a live one. */
+/**
+ * What an error from creating the lock file means. EEXIST: another process
+ * holds it. On Windows, creating a file whose previous copy is still being
+ * deleted (a lock just released) fails with EPERM, EBUSY or EACCES instead;
+ * that is the lock changing hands, so it is waited out like a held lock, not
+ * reported as a failure. Anything else is a real error.
+ */
+export function lockCreateError(code, platform = process.platform) {
+    if (code === "EEXIST")
+        return "held";
+    if (platform === "win32" && (code === "EPERM" || code === "EBUSY" || code === "EACCES"))
+        return "busy";
+    return "error";
+}
 export function tryAcquireLock(file, opts = {}) {
     const staleMs = opts.staleMs ?? DEFAULT_STALE_MS;
     const now = opts.now ?? Date.now;
@@ -289,8 +534,11 @@ export function tryAcquireLock(file, opts = {}) {
         }
     }
     catch (err) {
-        if (err.code !== "EEXIST")
+        const meaning = lockCreateError(err.code, opts.platform);
+        if (meaning === "error")
             throw err;
+        if (meaning === "busy")
+            return null; // being deleted as it changes hands: try again next poll
         let mtimeMs;
         try {
             mtimeMs = fs.statSync(file).mtimeMs;

package/dist/engine/report.js

@@ -1,6 +1,6 @@
 import fs from "node:fs";
 import path from "node:path";
-import { SHARED_CHROME_ROUTE, isEmbedKey, isWorthALook } from "./memory.js";
+import { SHARED_CHROME_ROUTE, isEmbedKey, isWorthALook, judgedMergesOf } from "./memory.js";
 import { sayVerification } from "./verify.js";
 import { feedForSession } from "./live.js";
 import { buildReplayHtml, evidenceFor } from "./replay.js";
@@ -503,6 +503,9 @@ export function generateReport(memory, oracleLog, extras, opts = {}) {
     if (extras?.policyAttributed) {
         lines.push(`| Errors caused by the tester's own write-policy blocks (not counted above) | ${extras.policyAttributed} |`);
     }
+    const judged = memory.dedupJudge?.describe() ?? (memory.dedupOff ? `the rule alone: the dedup judge was asked for and is off (${memory.dedupOff})` : null);
+    if (judged)
+        lines.push(`| Finding dedup | ${judged.replace(/\|/g, "/")} |`);
     lines.push(`| Elements exercised (informational — denominator grows with every state) | ${cov.elementsExercised}/${cov.elementsTotal} |`);
     lines.push(``);
     // ---- Page quality scores, worst first — the cross-page comparator. ----
@@ -601,6 +604,10 @@ export function generateReport(memory, oracleLog, extras, opts = {}) {
             lines.push(`- **Evidence:** \`${f.evidence}\``);
         lines.push(`- **Where:** \`${f.state}\` (${f.url})`);
         lines.push(`- **Seen in runs:** ${f.runs}`);
+        // A merge the model made is shown with what was filed, so a wrong one can be seen, and refiled as its own defect (ADR 4).
+        for (const m of judgedMergesOf(f)) {
+            lines.push(`- **Merged by the dedup judge** (p_same ${m.pSame.toFixed(2)}, ${m.at.slice(0, 10)}): [${m.severity}] ${m.title}, filed as ${m.category}${m.evidence ? ` with evidence \`${m.evidence}\`` : ""}`);
+        }
         // Only printed once somebody has re-tested it. A finding nobody has looked
         // at again says nothing here, which is the honest thing for it to say.
         if (f.verdict && f.verifiedAt) {