scenescout 3.15.0 → 3.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +87 -0
- package/README.md +70 -18
- package/dist/browsers.js +28 -0
- package/dist/check-run.js +191 -14
- package/dist/ci-run.js +268 -52
- package/dist/cli.js +107 -47
- package/dist/commands.js +3 -2
- package/dist/engine/baseline.js +377 -0
- package/dist/engine/brief.js +16 -7
- package/dist/engine/browser.js +1147 -286
- package/dist/engine/calibration.js +61 -30
- package/dist/engine/capture.js +164 -0
- package/dist/engine/check.js +244 -42
- package/dist/engine/ci-lanes.js +215 -0
- package/dist/engine/ci.js +136 -18
- package/dist/engine/claims.js +159 -3
- package/dist/engine/collector.js +561 -30
- package/dist/engine/crawl.js +49 -0
- package/dist/engine/design.js +281 -38
- package/dist/engine/export.js +877 -0
- package/dist/engine/fingerprint.js +92 -4
- package/dist/engine/flow.js +18 -6
- package/dist/engine/forms.js +181 -18
- package/dist/engine/journey.js +29 -1
- package/dist/engine/lane.js +13 -3
- package/dist/engine/launch.js +45 -6
- package/dist/engine/limits.js +7 -0
- package/dist/engine/live-page.js +49 -2
- package/dist/engine/live.js +4 -1
- package/dist/engine/memory.js +501 -47
- package/dist/engine/open.js +118 -0
- package/dist/engine/oracles.js +41 -1
- package/dist/engine/plain.js +268 -0
- package/dist/engine/png.js +127 -0
- package/dist/engine/policy.js +379 -9
- package/dist/engine/probes.js +3 -2
- package/dist/engine/profiles.js +45 -9
- package/dist/engine/project-folder.js +191 -0
- package/dist/engine/refresh.js +68 -3
- package/dist/engine/replay.js +63 -10
- package/dist/engine/report.js +241 -40
- package/dist/engine/request.js +317 -23
- package/dist/engine/sarif.js +120 -0
- package/dist/engine/settle.js +67 -0
- package/dist/engine/signed-in.js +256 -0
- package/dist/engine/status-pane-page.js +441 -0
- package/dist/engine/status-pane.js +128 -0
- package/dist/engine/tickets.js +671 -0
- package/dist/engine/unload.js +3 -2
- package/dist/export-run.js +633 -0
- package/dist/first-run.js +5 -0
- package/dist/installer.js +378 -8
- package/dist/intake.js +104 -0
- package/dist/login-run.js +250 -36
- package/dist/mcp-server.js +660 -65
- package/dist/playbook.js +5 -0
- package/dist/prompts.js +106 -0
- package/package.json +8 -5
- package/skills/scenescout/SKILL.md +49 -16
package/dist/engine/request.js
CHANGED
|
@@ -22,41 +22,143 @@
|
|
|
22
22
|
* Everything in this file is pure so it can be table-tested; the one
|
|
23
23
|
* `page.evaluate` lives in browser.ts.
|
|
24
24
|
*/
|
|
25
|
+
import { redactSecrets } from "./memory.js";
|
|
25
26
|
import { POLICY_REFUSAL_HEADER } from "./policy.js";
|
|
26
27
|
/** Methods a session may replay. Anything else is refused before it reaches the page. */
|
|
27
28
|
export const REPLAYABLE_METHODS = ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"];
|
|
28
29
|
/** Most of a response body that reaches the agent. A JSON list can be megabytes; the signature is in the first lines. */
|
|
29
30
|
export const BODY_MAX = 2000;
|
|
31
|
+
/** Most of a body one call can return when it asks for a part of it (`select`, or `offset`/`limit`). */
|
|
32
|
+
export const VIEW_MAX = 8000;
|
|
33
|
+
/** Most of a body the page hands back when a part of it was asked for: a select needs the whole document to parse. */
|
|
34
|
+
export const BODY_FETCH_MAX = 1_000_000;
|
|
35
|
+
/** Whether a view asks for anything other than the default first BODY_MAX characters. */
|
|
36
|
+
export function wantsView(view) {
|
|
37
|
+
return !!view && (view.select !== undefined || view.offset !== undefined || view.limit !== undefined);
|
|
38
|
+
}
|
|
39
|
+
/** The steps of a select path: "a.b[0].c" and "a.b.0.c" are the same three-then-one steps. */
|
|
40
|
+
export function selectSteps(select) {
|
|
41
|
+
return select
|
|
42
|
+
.trim()
|
|
43
|
+
.replace(/\[(\d+)\]/g, ".$1")
|
|
44
|
+
.split(".")
|
|
45
|
+
.filter((step) => step !== "");
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* The part of the body a view asks for, and the line that says what it is.
|
|
49
|
+
* A select parses the body as JSON and returns the value at the path,
|
|
50
|
+
* pretty-printed; a path that runs out says where and what keys were there
|
|
51
|
+
* instead. offset/limit then take a window of that text (or of the raw body
|
|
52
|
+
* with no select), and the note names the next offset when there is more.
|
|
53
|
+
*/
|
|
54
|
+
export function viewBody(text, full, view) {
|
|
55
|
+
let source = text;
|
|
56
|
+
let total = full;
|
|
57
|
+
let what = "the body";
|
|
58
|
+
if (view.select !== undefined) {
|
|
59
|
+
if (full > text.length) {
|
|
60
|
+
return {
|
|
61
|
+
body: "",
|
|
62
|
+
note: `select needs the whole body, and this one is ${full} characters, past the ${text.length} read. Page through it with offset and limit instead.`,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
let value;
|
|
66
|
+
try {
|
|
67
|
+
value = JSON.parse(text);
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
return { body: "", note: `select reads a JSON body, and this one is not JSON. Page through it with offset and limit instead.` };
|
|
71
|
+
}
|
|
72
|
+
const steps = selectSteps(view.select);
|
|
73
|
+
for (let i = 0; i < steps.length; i++) {
|
|
74
|
+
const step = steps[i];
|
|
75
|
+
const here = steps.slice(0, i).join(".") || "the top level";
|
|
76
|
+
if (Array.isArray(value)) {
|
|
77
|
+
const index = /^\d+$/.test(step) ? Number(step) : Number.NaN;
|
|
78
|
+
if (!(index < value.length))
|
|
79
|
+
return { body: "", note: `select ${JSON.stringify(view.select)}: no item ${JSON.stringify(step)} at ${here}, which is a list of ${value.length}.` };
|
|
80
|
+
value = value[index];
|
|
81
|
+
}
|
|
82
|
+
else if (value !== null && typeof value === "object") {
|
|
83
|
+
const record = value;
|
|
84
|
+
if (!Object.prototype.hasOwnProperty.call(record, step)) {
|
|
85
|
+
const keys = Object.keys(record);
|
|
86
|
+
return {
|
|
87
|
+
body: "",
|
|
88
|
+
note: `select ${JSON.stringify(view.select)}: no key ${JSON.stringify(step)} at ${here}. Keys there: ${keys.slice(0, 40).join(", ")}${keys.length > 40 ? ` … +${keys.length - 40}` : ""}.`,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
value = record[step];
|
|
92
|
+
}
|
|
93
|
+
else {
|
|
94
|
+
return {
|
|
95
|
+
body: "",
|
|
96
|
+
note: `select ${JSON.stringify(view.select)}: ${here} is ${value === null ? "null" : typeof value}, which has no ${JSON.stringify(step)}.`,
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
source = JSON.stringify(value, null, 2) ?? "undefined";
|
|
101
|
+
total = source.length;
|
|
102
|
+
what = `select ${JSON.stringify(view.select)}`;
|
|
103
|
+
}
|
|
104
|
+
const offset = Math.max(0, Math.floor(view.offset ?? 0));
|
|
105
|
+
const limit = Math.min(VIEW_MAX, Math.max(1, Math.floor(view.limit ?? (view.select !== undefined && view.offset === undefined ? VIEW_MAX : BODY_MAX))));
|
|
106
|
+
if (offset >= total && total > 0)
|
|
107
|
+
return { body: "", note: `${what}: offset ${offset} is past its end (${total} characters).` };
|
|
108
|
+
const end = Math.min(total, offset + limit);
|
|
109
|
+
const body = source.slice(offset, end);
|
|
110
|
+
const whole = offset === 0 && end === total;
|
|
111
|
+
const note = whole
|
|
112
|
+
? `${what}: all ${total} characters`
|
|
113
|
+
: `${what}: characters ${offset}–${end} of ${total}${end < total ? `; the next part is offset ${end}` : ""}`;
|
|
114
|
+
return { body, note };
|
|
115
|
+
}
|
|
30
116
|
/** Response headers worth reporting. "Identical response" means status, body AND headers, so the ones that commonly differ are kept. */
|
|
31
117
|
export const REPORTED_HEADERS = ["content-type", "content-length", "location", "www-authenticate", "retry-after", "x-request-id", "cache-control"];
|
|
32
118
|
/**
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
119
|
+
* Where a page, a crawl path, a plan step or a replayed call goes, given the
|
|
120
|
+
* URL the session was attached on. One rule for every tool:
|
|
121
|
+
*
|
|
122
|
+
* - A path resolves against the attached ORIGIN, never the attach URL's path.
|
|
123
|
+
* `/widgets` is `/widgets` whether the session attached on `/` or on
|
|
124
|
+
* `/things`, as URL rules and every browser read it; a bare `widgets` is
|
|
125
|
+
* read as `/widgets` too, so the page the session happens to be on never
|
|
126
|
+
* changes where a target goes.
|
|
127
|
+
* - A full URL on the attached origin is used as given.
|
|
128
|
+
* - Anything on another origin, or a scheme other than http(s), is refused:
|
|
129
|
+
* a run attached to one app must not be able to make its browser talk to
|
|
130
|
+
* another host just because a target was spelled as a full URL.
|
|
131
|
+
*
|
|
132
|
+
* `offOrigin` marks the refusal that is the fence, so a caller can word it as one.
|
|
37
133
|
*/
|
|
38
|
-
export function
|
|
39
|
-
const trimmed =
|
|
134
|
+
export function resolveTarget(attachUrl, target) {
|
|
135
|
+
const trimmed = target.trim();
|
|
40
136
|
if (!trimmed)
|
|
41
|
-
return { problem: "No path given. Pass a path such as /api/things, or a full URL on the attached origin." };
|
|
42
|
-
let
|
|
43
|
-
let
|
|
137
|
+
return { problem: "No path given. Pass a path such as /api/things, or a full URL on the attached origin.", offOrigin: false };
|
|
138
|
+
let resolved;
|
|
139
|
+
let origin;
|
|
44
140
|
try {
|
|
45
|
-
|
|
46
|
-
|
|
141
|
+
origin = new URL(attachUrl).origin;
|
|
142
|
+
resolved = new URL(trimmed, `${origin}/`);
|
|
47
143
|
}
|
|
48
144
|
catch {
|
|
49
|
-
return { problem: `Could not read ${JSON.stringify(trimmed)} as a path or a URL
|
|
145
|
+
return { problem: `Could not read ${JSON.stringify(trimmed)} as a path or a URL.`, offOrigin: false };
|
|
50
146
|
}
|
|
51
|
-
if (
|
|
52
|
-
return { problem: `Only http and https can be
|
|
147
|
+
if (resolved.protocol !== "http:" && resolved.protocol !== "https:") {
|
|
148
|
+
return { problem: `Only http and https can be reached; ${resolved.protocol} cannot.`, offOrigin: false };
|
|
53
149
|
}
|
|
54
|
-
if (
|
|
150
|
+
if (resolved.origin !== origin) {
|
|
55
151
|
return {
|
|
56
|
-
problem: `${
|
|
152
|
+
problem: `${resolved.origin} is not the origin this session is attached to (${origin}). A session talks to its own app only; attach another session to test another host.`,
|
|
153
|
+
offOrigin: true,
|
|
57
154
|
};
|
|
58
155
|
}
|
|
59
|
-
return { url:
|
|
156
|
+
return { url: resolved.toString() };
|
|
157
|
+
}
|
|
158
|
+
/** The URL scout_request calls: resolveTarget's rule, so a call and a page load never disagree on where a path goes. */
|
|
159
|
+
export function resolveRequestUrl(attachUrl, path) {
|
|
160
|
+
const out = resolveTarget(attachUrl, path);
|
|
161
|
+
return "url" in out ? out : { problem: out.problem };
|
|
60
162
|
}
|
|
61
163
|
/** The method, upper-cased, or the reason it cannot be replayed. */
|
|
62
164
|
export function resolveMethod(method) {
|
|
@@ -77,6 +179,7 @@ export function resolveMethod(method) {
|
|
|
77
179
|
*/
|
|
78
180
|
export function buildRequestScript(input) {
|
|
79
181
|
const { url, method, body, headers } = input;
|
|
182
|
+
const keep = input.keep ?? BODY_MAX * 2;
|
|
80
183
|
const init = { method, credentials: "include", headers };
|
|
81
184
|
if (body !== undefined && method !== "GET" && method !== "HEAD")
|
|
82
185
|
init.body = body;
|
|
@@ -84,7 +187,7 @@ export function buildRequestScript(input) {
|
|
|
84
187
|
` const res = await fetch(${JSON.stringify(url)}, ${JSON.stringify(init)});` +
|
|
85
188
|
` const text = await res.text();` +
|
|
86
189
|
` const headers = {}; res.headers.forEach((v, k) => { headers[k] = v; });` +
|
|
87
|
-
` return { status: res.status, statusText: res.statusText, headers, body: text.slice(0, ${
|
|
190
|
+
` return { status: res.status, statusText: res.statusText, headers, body: text.slice(0, ${keep}), full: text.length, ms: Date.now() - started, url: res.url }; })()`);
|
|
88
191
|
}
|
|
89
192
|
/** The headers to send: what the caller asked for, plus the app's own auth and a JSON content type when a body is present. */
|
|
90
193
|
export function requestHeaders(input) {
|
|
@@ -99,20 +202,63 @@ export function requestHeaders(input) {
|
|
|
99
202
|
out[name.toLowerCase()] = value;
|
|
100
203
|
return out;
|
|
101
204
|
}
|
|
205
|
+
/**
|
|
206
|
+
* The Authorization header to remember from a request the browser is sending,
|
|
207
|
+
* or null to leave the remembered one as it is. Every request the app sends to
|
|
208
|
+
* its own origin counts, reads included: an app that rotates its access token
|
|
209
|
+
* and then only reads would otherwise leave scout_request replaying the token
|
|
210
|
+
* of its last write, long expired. A request to another origin, or one sent
|
|
211
|
+
* from a frame of another origin, is skipped, so an embedded widget's bearer
|
|
212
|
+
* token is never replayed to the app; and so is a scout_request call's own
|
|
213
|
+
* request, so a credential the caller chose for one call does not become the
|
|
214
|
+
* session's. `frameUrl` is the sending frame's address, when it has one (a
|
|
215
|
+
* worker's request has none; about:blank and the like count as the app's).
|
|
216
|
+
*/
|
|
217
|
+
export function authToRemember(input) {
|
|
218
|
+
if (input.replay)
|
|
219
|
+
return null;
|
|
220
|
+
try {
|
|
221
|
+
const origin = new URL(input.baseUrl).origin;
|
|
222
|
+
if (new URL(input.url).origin !== origin)
|
|
223
|
+
return null;
|
|
224
|
+
if (input.frameUrl && /^https?:/i.test(input.frameUrl) && new URL(input.frameUrl).origin !== origin)
|
|
225
|
+
return null;
|
|
226
|
+
}
|
|
227
|
+
catch {
|
|
228
|
+
return null;
|
|
229
|
+
}
|
|
230
|
+
const entry = Object.entries(input.headers).find(([name]) => name.toLowerCase() === "authorization");
|
|
231
|
+
const value = entry?.[1];
|
|
232
|
+
return value && value.trim() ? value : null;
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* A line for a replay the server answered 401 while the page's own latest
|
|
236
|
+
* authorised request to the origin succeeded: the replayed credential is the
|
|
237
|
+
* likely cause, not the server's permissions. Empty otherwise. Statuses only,
|
|
238
|
+
* never the credential.
|
|
239
|
+
*/
|
|
240
|
+
export function staleCredentialNote(replayStatus, sentAuth, pageLast) {
|
|
241
|
+
if (replayStatus !== 401 || !sentAuth || !pageLast || pageLast.status < 200 || pageLast.status >= 300)
|
|
242
|
+
return "";
|
|
243
|
+
return (`\n⚠ The page's own latest authorised request to this origin got ${pageLast.status}, but this call got 401: ` +
|
|
244
|
+
`the replayed credential may be stale. Load a page that calls the API, then call again before reading this as a permission or sign-in problem.`);
|
|
245
|
+
}
|
|
102
246
|
/** The raw result of the in-page fetch, reduced to what the agent and the report need. */
|
|
103
|
-
export function toReplayResult(raw) {
|
|
247
|
+
export function toReplayResult(raw, view) {
|
|
104
248
|
const kept = {};
|
|
105
249
|
for (const name of REPORTED_HEADERS) {
|
|
106
250
|
const value = raw.headers[name];
|
|
107
251
|
if (value !== undefined)
|
|
108
252
|
kept[name] = value;
|
|
109
253
|
}
|
|
254
|
+
const part = wantsView(view) ? viewBody(raw.body, raw.full, view) : null;
|
|
110
255
|
return {
|
|
111
256
|
status: raw.status,
|
|
112
257
|
statusText: raw.statusText,
|
|
113
258
|
headers: kept,
|
|
114
|
-
body: raw.body.slice(0, BODY_MAX),
|
|
115
|
-
truncated: raw.full > BODY_MAX,
|
|
259
|
+
body: part ? part.body : raw.body.slice(0, BODY_MAX),
|
|
260
|
+
truncated: part ? false : raw.full > BODY_MAX,
|
|
261
|
+
...(part ? { view: part.note } : {}),
|
|
116
262
|
ms: raw.ms,
|
|
117
263
|
url: raw.url,
|
|
118
264
|
refusedByPolicy: raw.headers[POLICY_REFUSAL_HEADER] ?? null,
|
|
@@ -147,11 +293,159 @@ export function formatReplay(method, result) {
|
|
|
147
293
|
const headers = Object.entries(result.headers);
|
|
148
294
|
if (headers.length > 0)
|
|
149
295
|
lines.push(headers.map(([k, v]) => `${k}: ${v}`).join(" · "));
|
|
150
|
-
if (result.
|
|
151
|
-
lines.push("", result.body
|
|
296
|
+
if (result.view !== undefined) {
|
|
297
|
+
lines.push(`(${result.view})`, "", result.body.length > 0 ? result.body : "(nothing here)");
|
|
298
|
+
}
|
|
299
|
+
else if (result.body.length > 0) {
|
|
300
|
+
lines.push("", result.body +
|
|
301
|
+
(result.truncated
|
|
302
|
+
? `\n… truncated at ${BODY_MAX} characters — pass offset:${BODY_MAX} for the next part, or select:"a.b" for one value of a JSON body`
|
|
303
|
+
: ""));
|
|
152
304
|
}
|
|
153
305
|
else {
|
|
154
306
|
lines.push("", "(empty body)");
|
|
155
307
|
}
|
|
156
308
|
return lines.join("\n");
|
|
157
309
|
}
|
|
310
|
+
/**
|
|
311
|
+
* The data requests (fetch and XHR) the driven page made since its document
|
|
312
|
+
* loaded, newest last. A page that shows its empty state on a client-side tab
|
|
313
|
+
* switch while the API has data leaves two explanations — the list request
|
|
314
|
+
* failed, or it never ran — and only this tells them apart. A new document
|
|
315
|
+
* starts a new list; a client-side route change does not, so each entry
|
|
316
|
+
* keeps the route it was sent from. Bounded: past MAX the oldest go, and the
|
|
317
|
+
* listing says how many.
|
|
318
|
+
*/
|
|
319
|
+
export class PageRequests {
|
|
320
|
+
static MAX = 300;
|
|
321
|
+
loadedAt = 0;
|
|
322
|
+
loadedUrl = "";
|
|
323
|
+
entries = [];
|
|
324
|
+
dropped = 0;
|
|
325
|
+
byRequest = new WeakMap();
|
|
326
|
+
/** A new document began loading: the list starts again. */
|
|
327
|
+
loaded(url, at) {
|
|
328
|
+
this.loadedAt = at;
|
|
329
|
+
this.loadedUrl = url;
|
|
330
|
+
this.entries = [];
|
|
331
|
+
this.dropped = 0;
|
|
332
|
+
}
|
|
333
|
+
started(key, r) {
|
|
334
|
+
const entry = { method: r.method, url: r.url, route: r.route, atMs: Math.max(0, r.at - this.loadedAt) };
|
|
335
|
+
this.byRequest.set(key, entry);
|
|
336
|
+
this.entries.push(entry);
|
|
337
|
+
if (this.entries.length > PageRequests.MAX) {
|
|
338
|
+
this.entries.shift();
|
|
339
|
+
this.dropped += 1;
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
answered(key, status, at, refused) {
|
|
343
|
+
const entry = this.byRequest.get(key);
|
|
344
|
+
if (!entry || entry.status !== undefined)
|
|
345
|
+
return;
|
|
346
|
+
entry.status = status;
|
|
347
|
+
entry.ms = Math.max(0, at - this.loadedAt - entry.atMs);
|
|
348
|
+
if (refused)
|
|
349
|
+
entry.refused = true;
|
|
350
|
+
}
|
|
351
|
+
failed(key, reason, at, refused) {
|
|
352
|
+
const entry = this.byRequest.get(key);
|
|
353
|
+
if (!entry || entry.status !== undefined)
|
|
354
|
+
return;
|
|
355
|
+
entry.failed = reason || "failed";
|
|
356
|
+
entry.ms = Math.max(0, at - this.loadedAt - entry.atMs);
|
|
357
|
+
if (refused)
|
|
358
|
+
entry.refused = true;
|
|
359
|
+
}
|
|
360
|
+
/** Label the latest request to this URL as scout_request's own call. */
|
|
361
|
+
markReplay(method, url) {
|
|
362
|
+
const entry = [...this.entries].reverse().find((e) => e.method === method && e.url === url && !e.replay);
|
|
363
|
+
if (entry)
|
|
364
|
+
entry.replay = true;
|
|
365
|
+
}
|
|
366
|
+
get snapshot() {
|
|
367
|
+
return { loadedUrl: this.loadedUrl, loadedAt: this.loadedAt, entries: this.entries, dropped: this.dropped };
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
/** Most entries one listing prints by default, and at most. */
|
|
371
|
+
export const PAGE_REQUESTS_SHOWN = 40;
|
|
372
|
+
export const PAGE_REQUESTS_SHOWN_MAX = 200;
|
|
373
|
+
/**
|
|
374
|
+
* Query parameters whose value is a credential by its name, whatever the
|
|
375
|
+
* value looks like: `access_token`, `client_secret`, a signed URL's
|
|
376
|
+
* `X-Amz-Signature`, a one-time `code`.
|
|
377
|
+
*/
|
|
378
|
+
const SECRET_PARAM_RE = /token|secret|sig(nature)?$|^sig|password|passwd|api[-_]?key|^key$|auth|credential|session|^code$|otp/i;
|
|
379
|
+
/**
|
|
380
|
+
* A request's address as the listing prints it: the path on the page's
|
|
381
|
+
* origin, the whole URL elsewhere. A query parameter named like a
|
|
382
|
+
* credential has its value replaced before anything else, then the rest
|
|
383
|
+
* passes the same redaction as an oracle's violation; it is cut only after.
|
|
384
|
+
*/
|
|
385
|
+
export function shownUrl(url, origin) {
|
|
386
|
+
let shown = url;
|
|
387
|
+
try {
|
|
388
|
+
const u = new URL(url);
|
|
389
|
+
let hidden = 0;
|
|
390
|
+
for (const name of [...new Set(u.searchParams.keys())]) {
|
|
391
|
+
if (SECRET_PARAM_RE.test(name)) {
|
|
392
|
+
u.searchParams.set(name, "redacted");
|
|
393
|
+
hidden += 1;
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
const search = hidden > 0 ? u.search.replace(/=redacted\b/g, "=[redacted]") : u.search;
|
|
397
|
+
shown = (u.origin === origin ? "" : u.origin) + u.pathname + search;
|
|
398
|
+
}
|
|
399
|
+
catch {
|
|
400
|
+
// Not a URL we can shorten; shown whole, and redacted below.
|
|
401
|
+
}
|
|
402
|
+
const redacted = redactSecrets(shown);
|
|
403
|
+
return redacted.length > 300 ? `${redacted.slice(0, 300)}…` : redacted;
|
|
404
|
+
}
|
|
405
|
+
/**
|
|
406
|
+
* The listing scout_network returns: one line per request, oldest first, the
|
|
407
|
+
* newest `limit` of them, optionally only those whose address contains a
|
|
408
|
+
* string. Query values that look like credentials are redacted the way an
|
|
409
|
+
* oracle's violation is, because the listing is printed and may be quoted in
|
|
410
|
+
* a finding.
|
|
411
|
+
*/
|
|
412
|
+
export function formatPageRequests(log, opts) {
|
|
413
|
+
if (!log.loadedUrl)
|
|
414
|
+
return "No page has loaded in this session yet: navigate first.";
|
|
415
|
+
let origin = "";
|
|
416
|
+
let loadedRoute = log.loadedUrl;
|
|
417
|
+
try {
|
|
418
|
+
const u = new URL(log.loadedUrl);
|
|
419
|
+
origin = u.origin;
|
|
420
|
+
loadedRoute = u.pathname;
|
|
421
|
+
}
|
|
422
|
+
catch {
|
|
423
|
+
// Kept as given.
|
|
424
|
+
}
|
|
425
|
+
const limit = Math.min(PAGE_REQUESTS_SHOWN_MAX, Math.max(1, Math.floor(opts.limit ?? PAGE_REQUESTS_SHOWN)));
|
|
426
|
+
const needle = opts.contains?.trim().toLowerCase() ?? "";
|
|
427
|
+
const matching = log.entries.filter((e) => !needle || shownUrl(e.url, origin).toLowerCase().includes(needle));
|
|
428
|
+
const shown = matching.slice(-limit);
|
|
429
|
+
const ago = Math.max(0, Math.round((opts.now - log.loadedAt) / 1000));
|
|
430
|
+
const head = `DATA REQUESTS (fetch/XHR) since this page loaded — ${shownUrl(log.loadedUrl, origin)}, ${ago}s ago: ` +
|
|
431
|
+
`${matching.length}${needle ? ` matching ${JSON.stringify(opts.contains)}` : ""}` +
|
|
432
|
+
(shown.length < matching.length ? `, the newest ${shown.length} shown` : "") +
|
|
433
|
+
(log.dropped > 0 ? ` (${log.dropped} older ones no longer kept)` : "");
|
|
434
|
+
if (shown.length === 0) {
|
|
435
|
+
return `${head}.\nNone${needle ? " matching" : ""}: if the page shows data it fetched, it fetched it before this load or without fetch/XHR (a server-rendered page, a WebSocket).`;
|
|
436
|
+
}
|
|
437
|
+
const lines = shown.map((e) => {
|
|
438
|
+
const outcome = e.status !== undefined
|
|
439
|
+
? `${e.status}${e.ms !== undefined ? ` · ${e.ms} ms` : ""}`
|
|
440
|
+
: e.failed !== undefined
|
|
441
|
+
? `failed: ${e.failed}`
|
|
442
|
+
: "pending (no answer yet)";
|
|
443
|
+
const notes = [
|
|
444
|
+
e.refused ? "refused by the write policy, never reached the server" : "",
|
|
445
|
+
e.replay ? "sent by scout_request" : "",
|
|
446
|
+
e.route && e.route !== loadedRoute ? `on ${e.route}` : "",
|
|
447
|
+
].filter(Boolean);
|
|
448
|
+
return ` +${(e.atMs / 1000).toFixed(1)}s ${e.method} ${shownUrl(e.url, origin)} → ${outcome}${notes.length ? ` (${notes.join("; ")})` : ""}`;
|
|
449
|
+
});
|
|
450
|
+
return [`${head}:`, ...lines].join("\n");
|
|
451
|
+
}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where a SARIF result points. GitHub code scanning keeps a result only when
|
|
3
|
+
* its location is a file in the repository, so a result about a page of the
|
|
4
|
+
* running app cannot point at the page itself. Each result's physical
|
|
5
|
+
* location is a repository file instead, and the route travels beside it: in
|
|
6
|
+
* a logical location, in the result's properties and in its message.
|
|
7
|
+
*
|
|
8
|
+
* - An issue a saved flow raised points at that flow's file.
|
|
9
|
+
* - Anything else points at the anchor: --sarif-file-anchor when given, else
|
|
10
|
+
* the workflow file that is running (GITHUB_WORKFLOW_REF), else
|
|
11
|
+
* package.json when the repository has one, else README.md.
|
|
12
|
+
*
|
|
13
|
+
* Paths are relative to the repository root: GITHUB_WORKSPACE when it is set,
|
|
14
|
+
* else the project directory. Pure, so check-test can table-test it.
|
|
15
|
+
*/
|
|
16
|
+
import path from "node:path";
|
|
17
|
+
/** The anchors tried, in order, when there is no option and no workflow. */
|
|
18
|
+
export const SARIF_ANCHOR_FALLBACKS = ["package.json", "README.md"];
|
|
19
|
+
/** A repository-relative path with forward slashes, or an error: what --sarif-file-anchor accepts. */
|
|
20
|
+
export function checkSarifAnchor(raw) {
|
|
21
|
+
const value = raw
|
|
22
|
+
.trim()
|
|
23
|
+
.replace(/\\/g, "/")
|
|
24
|
+
.replace(/^(\.\/)+/, "");
|
|
25
|
+
if (!value)
|
|
26
|
+
return { ok: false, error: "--sarif-file-anchor needs a file, relative to the repository root" };
|
|
27
|
+
if (value.startsWith("/") || /^[A-Za-z]:\//.test(value))
|
|
28
|
+
return { ok: false, error: "--sarif-file-anchor is a path relative to the repository root, not an absolute path" };
|
|
29
|
+
const parts = value.split("/");
|
|
30
|
+
if (parts.includes(".."))
|
|
31
|
+
return { ok: false, error: "--sarif-file-anchor must stay inside the repository: no .. in the path" };
|
|
32
|
+
if (value.endsWith("/"))
|
|
33
|
+
return { ok: false, error: "--sarif-file-anchor names a file, not a directory" };
|
|
34
|
+
return { ok: true, value: parts.filter((p) => p !== "." && p !== "").join("/") };
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* The workflow file in GITHUB_WORKFLOW_REF (`owner/repo/.github/workflows/x.yml@refs/heads/main`),
|
|
38
|
+
* or null when the variable is unset or is not of that shape.
|
|
39
|
+
*/
|
|
40
|
+
export function workflowFileOf(ref) {
|
|
41
|
+
if (!ref)
|
|
42
|
+
return null;
|
|
43
|
+
const parts = ref.split("/");
|
|
44
|
+
if (parts.length < 3 || !parts[0] || !parts[1])
|
|
45
|
+
return null;
|
|
46
|
+
// The ref after the first "@" may hold "@" itself (a branch named fix@2); the workflow's path comes before it.
|
|
47
|
+
const rest = parts.slice(2).join("/");
|
|
48
|
+
const at = rest.indexOf("@");
|
|
49
|
+
const file = at >= 0 ? rest.slice(0, at) : rest;
|
|
50
|
+
return checkSarifAnchor(file).ok ? file : null;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* The anchor file, where it came from, and a warning when the file it should
|
|
54
|
+
* have been is not in the repository. Candidates are tried in order: the
|
|
55
|
+
* option, the workflow file, then each fallback; the first that exists wins.
|
|
56
|
+
* When none does, the first candidate is kept so the SARIF is still written,
|
|
57
|
+
* and the warning says code scanning will drop its results.
|
|
58
|
+
* `exists` is asked about repository-relative paths.
|
|
59
|
+
*/
|
|
60
|
+
export function resolveSarifAnchor(o) {
|
|
61
|
+
const workflow = workflowFileOf(o.env.GITHUB_WORKFLOW_REF);
|
|
62
|
+
const candidates = [
|
|
63
|
+
...(o.option !== undefined ? [{ file: o.option, source: "option" }] : []),
|
|
64
|
+
...(workflow ? [{ file: workflow, source: "workflow" }] : []),
|
|
65
|
+
...SARIF_ANCHOR_FALLBACKS.map((file) => ({ file, source: "fallback" })),
|
|
66
|
+
];
|
|
67
|
+
const describe = (c) => c.source === "option" ? `--sarif-file-anchor ${c.file}` : c.source === "workflow" ? `the workflow file ${c.file}` : c.file;
|
|
68
|
+
const chosen = candidates.find((c) => o.exists(c.file));
|
|
69
|
+
// Fallbacks are tried quietly; only a file that was asked for, or the workflow running, is missed out loud.
|
|
70
|
+
const missed = candidates.slice(0, chosen ? candidates.indexOf(chosen) : candidates.length).filter((c) => c.source !== "fallback");
|
|
71
|
+
if (!chosen) {
|
|
72
|
+
const first = candidates[0];
|
|
73
|
+
return {
|
|
74
|
+
...first,
|
|
75
|
+
warning: `SARIF: no anchor file exists in the repository (tried ${candidates.map(describe).join(", ")}); results point at ${first.file}, and code scanning will drop them`,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
if (missed.length === 0)
|
|
79
|
+
return chosen;
|
|
80
|
+
return { ...chosen, warning: `SARIF: ${missed.map(describe).join(" and ")} is not in the repository; results point at ${chosen.file} instead` };
|
|
81
|
+
}
|
|
82
|
+
/** The repository root the paths are relative to: the Actions checkout when there is one, else the project. */
|
|
83
|
+
export function repositoryRoot(env, projectDir) {
|
|
84
|
+
return env.GITHUB_WORKSPACE || projectDir;
|
|
85
|
+
}
|
|
86
|
+
/** `target` relative to `root` with forward slashes, or null when it is outside it. */
|
|
87
|
+
export function repoRelative(root, target) {
|
|
88
|
+
const rel = path.relative(path.resolve(root), path.resolve(target));
|
|
89
|
+
if (rel === "")
|
|
90
|
+
return "";
|
|
91
|
+
if (rel.startsWith("..") || path.isAbsolute(rel))
|
|
92
|
+
return null;
|
|
93
|
+
return rel.split(path.sep).join("/");
|
|
94
|
+
}
|
|
95
|
+
/** The file a result points at: its flow's file when a flow raised it and the flows are in the repository, else the anchor. */
|
|
96
|
+
export function sarifFileFor(files, flowFile) {
|
|
97
|
+
return flowFile && files.flowsDir !== undefined ? (files.flowsDir ? `${files.flowsDir}/${flowFile}` : flowFile) : files.anchor;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* One SARIF location: a repository file code scanning can resolve, and the
|
|
101
|
+
* route as a logical location of kind "resource".
|
|
102
|
+
*/
|
|
103
|
+
export function sarifLocation(file, route, message) {
|
|
104
|
+
return {
|
|
105
|
+
physicalLocation: { artifactLocation: { uri: file } },
|
|
106
|
+
logicalLocations: [{ kind: "resource", name: route, fullyQualifiedName: route }],
|
|
107
|
+
...(message ? { message: { text: message } } : {}),
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Everything a run's SARIF needs to know about the repository, from its
|
|
112
|
+
* option, its environment and the directories it read. `exists` takes an
|
|
113
|
+
* absolute path.
|
|
114
|
+
*/
|
|
115
|
+
export function sarifFilesFor(o) {
|
|
116
|
+
const root = repositoryRoot(o.env, o.projectDir);
|
|
117
|
+
const { file, source, warning } = resolveSarifAnchor({ option: o.option, env: o.env, exists: (f) => o.exists(path.join(root, f)) });
|
|
118
|
+
const flowsDir = o.flowsDir ? repoRelative(root, o.flowsDir) : null;
|
|
119
|
+
return { anchor: file, source, ...(warning ? { warning } : {}), ...(flowsDir !== null ? { flowsDir } : {}) };
|
|
120
|
+
}
|
package/dist/engine/settle.js
CHANGED
|
@@ -51,6 +51,73 @@ export function shouldKeepWaiting(state) {
|
|
|
51
51
|
// NEXT action, which is then blamed for it.
|
|
52
52
|
return Math.min(state.sinceLastStartMs, state.elapsedMs) < QUIET_MS;
|
|
53
53
|
}
|
|
54
|
+
/**
|
|
55
|
+
* Which requests are still in flight, for the wait above.
|
|
56
|
+
*
|
|
57
|
+
* Counting the context's request events up and down is not enough. A request
|
|
58
|
+
* that has had its response headers but not its whole body when its document
|
|
59
|
+
* goes away (the page is left, or a frame is removed or moves on) gets neither
|
|
60
|
+
* a "finished" nor a "failed" event, so a plain counter never comes back down
|
|
61
|
+
* and every later wait runs to its cap. Each request is therefore held with the
|
|
62
|
+
* frame that sent it and the order it started in, and is dropped when it ends,
|
|
63
|
+
* when its frame is gone, or when its frame commits a navigation that started
|
|
64
|
+
* after it did. A same-document navigation (history.pushState) sends no
|
|
65
|
+
* navigation request, so it drops nothing — unless a navigation that answered
|
|
66
|
+
* without a new document (a 204, a download) is still pending for that frame,
|
|
67
|
+
* in which case the requests started before it are dropped early. Generic over the request and frame
|
|
68
|
+
* types so it is tested without a browser.
|
|
69
|
+
*/
|
|
70
|
+
export class InFlightRequests {
|
|
71
|
+
open = new Map();
|
|
72
|
+
/** Per frame, the order of the latest navigation request it sent that has not yet committed. */
|
|
73
|
+
pendingNavigation = new Map();
|
|
74
|
+
order = 0;
|
|
75
|
+
/** A request started. `frame` is undefined for one no frame sent (a worker's). */
|
|
76
|
+
started(request, frame, isNavigation) {
|
|
77
|
+
const order = ++this.order;
|
|
78
|
+
this.open.set(request, { frame, order });
|
|
79
|
+
if (isNavigation && frame !== undefined)
|
|
80
|
+
this.pendingNavigation.set(frame, order);
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* A request finished or failed. One already dropped is ignored, so it cannot
|
|
84
|
+
* be counted out twice. A navigation that failed never commits, so it stops
|
|
85
|
+
* being the one a later commit of its frame is measured against. A finished
|
|
86
|
+
* one may still be about to commit (its body can end first), so it stays.
|
|
87
|
+
*/
|
|
88
|
+
ended(request, failed = false) {
|
|
89
|
+
const held = this.open.get(request);
|
|
90
|
+
this.open.delete(request);
|
|
91
|
+
if (failed && held?.frame !== undefined && this.pendingNavigation.get(held.frame) === held.order)
|
|
92
|
+
this.pendingNavigation.delete(held.frame);
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* A frame committed a navigation. Requests its old document sent before the
|
|
96
|
+
* navigation began are dropped; the navigation itself and anything started
|
|
97
|
+
* after it stay. Without a navigation request (a same-document change) nothing is dropped.
|
|
98
|
+
*/
|
|
99
|
+
navigated(frame) {
|
|
100
|
+
const navigation = this.pendingNavigation.get(frame);
|
|
101
|
+
if (navigation === undefined)
|
|
102
|
+
return;
|
|
103
|
+
this.pendingNavigation.delete(frame);
|
|
104
|
+
for (const [request, held] of this.open)
|
|
105
|
+
if (held.frame === frame && held.order < navigation)
|
|
106
|
+
this.open.delete(request);
|
|
107
|
+
}
|
|
108
|
+
/** Frames that are gone (detached, or their page closed): every request they sent is dropped. */
|
|
109
|
+
gone(isGone) {
|
|
110
|
+
for (const [request, held] of this.open)
|
|
111
|
+
if (held.frame !== undefined && isGone(held.frame))
|
|
112
|
+
this.open.delete(request);
|
|
113
|
+
for (const frame of [...this.pendingNavigation.keys()])
|
|
114
|
+
if (isGone(frame))
|
|
115
|
+
this.pendingNavigation.delete(frame);
|
|
116
|
+
}
|
|
117
|
+
get count() {
|
|
118
|
+
return this.open.size;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
54
121
|
/** A pace a session asked for, clamped. Negative or absurd values are a typo, not an instruction. */
|
|
55
122
|
export const PACE_MAX_MS = 60_000;
|
|
56
123
|
export function normalizePace(paceMs) {
|