scenescout 3.21.3 → 3.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +5 -1
- package/dist/check-run.js +62 -1
- package/dist/ci-run.js +259 -36
- package/dist/cli.js +19 -4
- package/dist/engine/brief.js +35 -3
- package/dist/engine/check-replay.js +12 -2
- package/dist/engine/check-report.js +503 -0
- package/dist/engine/check.js +9 -0
- package/dist/engine/ci-lanes.js +28 -12
- package/dist/engine/ci.js +39 -1
- package/dist/engine/flow.js +28 -8
- package/dist/engine/from-run.js +649 -0
- package/dist/engine/memory.js +123 -0
- package/dist/engine/report.js +26 -0
- package/dist/mcp-server.js +63 -5
- package/package.json +3 -2
- package/skills/scenescout/SKILL.md +2 -2
package/dist/engine/check.js
CHANGED
|
@@ -503,6 +503,7 @@ export const CHECK_OPTION_NAMES = [
|
|
|
503
503
|
"sarif-file-anchor",
|
|
504
504
|
"record",
|
|
505
505
|
"video",
|
|
506
|
+
"template",
|
|
506
507
|
];
|
|
507
508
|
/** Options that may be given alone, meaning on: `--record`, as well as `--record on` and `--record=off`. */
|
|
508
509
|
const SWITCH_OPTIONS = new Set(["record", "video"]);
|
|
@@ -685,6 +686,12 @@ export function parseCheckArgs(args, cwd) {
|
|
|
685
686
|
if (videoRaw !== undefined && !SWITCH_ON.includes(videoRaw) && !SWITCH_OFF.includes(videoRaw)) {
|
|
686
687
|
return { ok: false, error: "--video is on or off, or given alone for on" };
|
|
687
688
|
}
|
|
689
|
+
const template = flags.get("template");
|
|
690
|
+
if (template !== undefined && template.trim() === "")
|
|
691
|
+
return { ok: false, error: "--template needs the path of a report template (JSON)" };
|
|
692
|
+
if (template !== undefined && recordRaw !== undefined && SWITCH_OFF.includes(recordRaw)) {
|
|
693
|
+
return { ok: false, error: "--template renders the report from a recorded check's frames, so it cannot be used with --record off" };
|
|
694
|
+
}
|
|
688
695
|
const resolve = (p) => resolveArgPath(cwd, p);
|
|
689
696
|
return {
|
|
690
697
|
ok: true,
|
|
@@ -713,6 +720,7 @@ export function parseCheckArgs(args, cwd) {
|
|
|
713
720
|
...(anchor ? { sarifFileAnchor: anchor.value } : {}),
|
|
714
721
|
...(recordRaw !== undefined ? { record: SWITCH_ON.includes(recordRaw) } : {}),
|
|
715
722
|
...(videoRaw !== undefined && SWITCH_ON.includes(videoRaw) ? { video: true } : {}),
|
|
723
|
+
...(template !== undefined ? { template: resolve(template) } : {}),
|
|
716
724
|
},
|
|
717
725
|
};
|
|
718
726
|
}
|
|
@@ -1178,5 +1186,6 @@ export function toSummaryJson(result, toolVersion) {
|
|
|
1178
1186
|
worthALook: result.worthALook,
|
|
1179
1187
|
// Routes a check with no session was sent to sign-in from: a coverage gap. Absent when the check had a session.
|
|
1180
1188
|
...(result.needsSignIn ? { needsSignIn: result.needsSignIn } : {}),
|
|
1189
|
+
...(result.testReport ? { testReport: result.testReport } : {}),
|
|
1181
1190
|
};
|
|
1182
1191
|
}
|
package/dist/engine/ci-lanes.js
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* and how the lanes' endings become the run's. The loop that runs them is
|
|
9
9
|
* src/ci-run.ts; the shared budget is engine/ci.ts's. Why: ADR 20.
|
|
10
10
|
*/
|
|
11
|
-
import { LANE_RULES, planLanes } from "./brief.js";
|
|
11
|
+
import { LANE_RULES, planLanes, replayBriefs } from "./brief.js";
|
|
12
12
|
import { CI_TOOLS } from "./ci.js";
|
|
13
13
|
/** The session the run attaches first. It plans, holds no lane, and writes the report once every lane is done. */
|
|
14
14
|
export const PLANNER_SESSION = "default";
|
|
@@ -88,6 +88,10 @@ export function crawlNotes(text) {
|
|
|
88
88
|
export function crawlFoundNothing(text) {
|
|
89
89
|
return /^(?:\[session [^\]]*\]\s*)?(Nothing to crawl|Nothing new to crawl|No routes to crawl yet)/m.test(text);
|
|
90
90
|
}
|
|
91
|
+
/** The routes a run has to plan with: the page it attached on, then every route the planning crawl listed. */
|
|
92
|
+
export function plannedRoutes(target, notes) {
|
|
93
|
+
return [...new Set([routeOf(target), ...notes.keys()])];
|
|
94
|
+
}
|
|
91
95
|
/** The path a target URL opens, as a route: the planning crawl never lists the page the run attached on. */
|
|
92
96
|
function routeOf(url) {
|
|
93
97
|
try {
|
|
@@ -119,12 +123,12 @@ export function laneSessions(names) {
|
|
|
119
123
|
* run has nothing to split, and explores in one loop, saying why.
|
|
120
124
|
*/
|
|
121
125
|
export function planCiLanes(o) {
|
|
122
|
-
const routes =
|
|
126
|
+
const routes = plannedRoutes(o.target, o.notes);
|
|
123
127
|
if (o.count < 2)
|
|
124
128
|
return { lanes: [], oneLoop: "one lane was asked for" };
|
|
125
129
|
if (o.notes.size === 0 && o.planningFailed)
|
|
126
130
|
return { lanes: [], oneLoop: `${o.planningFailed}, so there was nothing to split` };
|
|
127
|
-
const briefs = planLanes(routes, o.count, { goal: o.focus, mode: o.mode });
|
|
131
|
+
const briefs = planLanes(routes, o.count, { goal: o.focus, mode: o.mode, ...(o.order ? { order: o.order } : {}) });
|
|
128
132
|
if (briefs.length < 2) {
|
|
129
133
|
const where = briefs[0]?.modules[0];
|
|
130
134
|
return {
|
|
@@ -132,16 +136,27 @@ export function planCiLanes(o) {
|
|
|
132
136
|
oneLoop: `the planning crawl found ${routes.length} route(s), all in ${where ? `one module (${where})` : "no module"}, so there was nothing to split`,
|
|
133
137
|
};
|
|
134
138
|
}
|
|
139
|
+
return { lanes: asCiLanes(briefs, o.target, o.notes) };
|
|
140
|
+
}
|
|
141
|
+
function asCiLanes(briefs, target, notes) {
|
|
135
142
|
const sessions = laneSessions(briefs.map((b) => b.lane));
|
|
136
|
-
return {
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
143
|
+
return briefs.map((b, i) => ({
|
|
144
|
+
...b,
|
|
145
|
+
session: sessions[i],
|
|
146
|
+
// On the target's origin whatever the route says: a path written `//host/x` must not become another host.
|
|
147
|
+
url: `${new URL(target).origin}${b.landing.startsWith("/") ? "" : "/"}${b.landing}`,
|
|
148
|
+
crawl: b.routes.flatMap((r) => notes.get(r) ?? []).slice(0, LIST_MAX),
|
|
149
|
+
}));
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* A replayed run's lanes (from-run.ts replayPlan): one per session the
|
|
153
|
+
* recorded run had, each with its routes in recorded order, however many lanes
|
|
154
|
+
* were asked for. With fewer than two the run replays in one loop.
|
|
155
|
+
*/
|
|
156
|
+
export function planReplayLanes(o) {
|
|
157
|
+
if (o.lanes.length < 2)
|
|
158
|
+
return { lanes: [], oneLoop: `the recorded run had ${o.lanes.length === 1 ? "one session" : "no steps"}, so it is replayed in one loop` };
|
|
159
|
+
return { lanes: asCiLanes(replayBriefs(o.lanes, o.focus), o.target, new Map()) };
|
|
145
160
|
}
|
|
146
161
|
// ── what a lane is told ─────────────────────────────────────────────────────
|
|
147
162
|
/**
|
|
@@ -171,6 +186,7 @@ export function ciLaneKickoff(o) {
|
|
|
171
186
|
`Target: ${o.url}`,
|
|
172
187
|
`Your lane: ${lane.objective}`,
|
|
173
188
|
`Your routes (${lane.routes.length}): ${lane.routes.slice(0, LIST_MAX).join(", ")}${lane.routes.length > LIST_MAX ? ` … and ${lane.routes.length - LIST_MAX} more` : ""}`,
|
|
189
|
+
...(o.fromRun ?? []),
|
|
174
190
|
`Your browser is on ${routeOf(lane.url)}.`,
|
|
175
191
|
lane.crawl.length > 0 ? `What the planning crawl saw on your routes:\n${lane.crawl.map((l) => ` ${l.trim()}`).join("\n")}` : "",
|
|
176
192
|
`Project directory (for scout_scan): ${o.projectDir}`,
|
package/dist/engine/ci.js
CHANGED
|
@@ -16,6 +16,7 @@ import { markdownCell } from "./check.js";
|
|
|
16
16
|
import { parseLimitFlag } from "./limits.js";
|
|
17
17
|
import { isWorthALook, redactSecrets } from "./memory.js";
|
|
18
18
|
import { SARIF_ANCHOR_FALLBACKS, checkSarifAnchor, sarifLocation } from "./sarif.js";
|
|
19
|
+
import { fromRunLine, resolveFromRun } from "./from-run.js";
|
|
19
20
|
// ── options ─────────────────────────────────────────────────────────────────
|
|
20
21
|
export const PROVIDERS = ["anthropic", "openai"];
|
|
21
22
|
/** Where each provider's key is read from. Nothing else is: no flag, no file, no input. */
|
|
@@ -93,6 +94,9 @@ export const CI_OPTION_NAMES = [
|
|
|
93
94
|
"show",
|
|
94
95
|
"compare-url",
|
|
95
96
|
"dedup",
|
|
97
|
+
"from-run",
|
|
98
|
+
"from-run-mode",
|
|
99
|
+
"from-run-turns-per-page",
|
|
96
100
|
"sarif-file-anchor",
|
|
97
101
|
];
|
|
98
102
|
/** The longest --show description: it becomes a line of the model's prompt. */
|
|
@@ -115,7 +119,8 @@ export function checkBaseUrl(raw) {
|
|
|
115
119
|
return { ok: true, url: u.toString().replace(/\/+$/, "") };
|
|
116
120
|
return { ok: false, error: "--base-url must be https (plain http only to 127.0.0.1 or localhost): the API key is sent to it" };
|
|
117
121
|
}
|
|
118
|
-
|
|
122
|
+
/** `env` is read for the earlier run alone (SCENESCOUT_FROM_RUN, SCENESCOUT_FROM_RUN_MODE), since a flag wins over it. */
|
|
123
|
+
export function parseCiArgs(args, cwd, env = {}) {
|
|
119
124
|
const positional = [];
|
|
120
125
|
const flags = new Map();
|
|
121
126
|
for (let i = 0; i < args.length; i++) {
|
|
@@ -271,6 +276,13 @@ export function parseCiArgs(args, cwd) {
|
|
|
271
276
|
const anchor = flags.has("sarif-file-anchor") ? checkSarifAnchor(flags.get("sarif-file-anchor")) : undefined;
|
|
272
277
|
if (anchor && !anchor.ok)
|
|
273
278
|
return anchor;
|
|
279
|
+
const runFlags = { path: flags.get("from-run"), mode: flags.get("from-run-mode"), turnsPerPage: flags.get("from-run-turns-per-page") };
|
|
280
|
+
if (show && (runFlags.path !== undefined || runFlags.mode !== undefined || runFlags.turnsPerPage !== undefined))
|
|
281
|
+
return { ok: false, error: "--from-run starts an exploration from an earlier run, and --show explores nothing: give one or the other" };
|
|
282
|
+
// A run asked to show an element explores nothing, so an earlier run named in the environment does not apply to it.
|
|
283
|
+
const fromRun = show ? { ok: true } : resolveFromRun(runFlags, env);
|
|
284
|
+
if (!fromRun.ok)
|
|
285
|
+
return fromRun;
|
|
274
286
|
const resolve = (p) => (p.startsWith("/") || /^[A-Za-z]:[\\/]/.test(p) ? p : `${cwd.replace(/[\\/]$/, "")}/${p}`);
|
|
275
287
|
return {
|
|
276
288
|
ok: true,
|
|
@@ -296,6 +308,7 @@ export function parseCiArgs(args, cwd) {
|
|
|
296
308
|
...(navTimeout.value !== undefined ? { navTimeoutMs: navTimeout.value } : {}),
|
|
297
309
|
dedup: dedup,
|
|
298
310
|
...(anchor ? { sarifFileAnchor: anchor.value } : {}),
|
|
311
|
+
...(fromRun.fromRun ? { fromRun: { ...fromRun.fromRun, path: resolve(fromRun.fromRun.path), given: fromRun.fromRun.path } } : {}),
|
|
299
312
|
},
|
|
300
313
|
};
|
|
301
314
|
}
|
|
@@ -691,6 +704,7 @@ export function ciKickoff(o) {
|
|
|
691
704
|
return [
|
|
692
705
|
`Run an exploratory test session following the method.`,
|
|
693
706
|
`Target: ${o.url}`,
|
|
707
|
+
...(o.fromRunLines ?? []),
|
|
694
708
|
`Project directory (for scout_scan): ${o.projectDir}`,
|
|
695
709
|
`Level: ${o.level}`,
|
|
696
710
|
`Write mode: ${o.mode}`,
|
|
@@ -760,6 +774,7 @@ export function ciSummaryMarkdown(r, secrets = []) {
|
|
|
760
774
|
`| Model | ${r.provider} ${cell(r.model, secrets)}, effort ${r.effort} |`,
|
|
761
775
|
...(r.lanes ? [`| Lanes | ${lanesCell(r.lanes, secrets)} |`] : []),
|
|
762
776
|
...(r.dedup ? [`| Finding dedup | ${dedupLine(r.dedup)} |`] : []),
|
|
777
|
+
...(r.fromRun ? [`| Started from | ${cell(fromRunLine(r.fromRun), secrets)} |`] : []),
|
|
763
778
|
`| Usage | ${usageLine(r.spend, r.model, r.endedAt, r.price)} |`,
|
|
764
779
|
``,
|
|
765
780
|
];
|
|
@@ -785,6 +800,27 @@ export function ciSummaryMarkdown(r, secrets = []) {
|
|
|
785
800
|
lines.push(r.capture ? `The pictures are in shots/.` : `The full report, with repro steps and the gap ledger, is report.md.`, ``);
|
|
786
801
|
return lines.join("\n");
|
|
787
802
|
}
|
|
803
|
+
/** A run record with every text a page or a person wrote in it passed through `clean`, as everything else ci.json holds is. */
|
|
804
|
+
function cleanRecord(r, clean) {
|
|
805
|
+
const all = (xs) => xs.map(clean);
|
|
806
|
+
return {
|
|
807
|
+
...r,
|
|
808
|
+
knownRoutes: all(r.knownRoutes),
|
|
809
|
+
visited: all(r.visited),
|
|
810
|
+
lanes: r.lanes.map((l) => ({ session: l.session, routes: all(l.routes) })),
|
|
811
|
+
steps: r.steps.map((s) => ({ ...s, route: clean(s.route), ...(s.target !== undefined ? { target: clean(s.target) } : {}) })),
|
|
812
|
+
left: r.left.map((l) => ({
|
|
813
|
+
...l,
|
|
814
|
+
route: clean(l.route),
|
|
815
|
+
unexercised: all(l.unexercised),
|
|
816
|
+
forms: all(l.forms),
|
|
817
|
+
unchosen: l.unchosen.map((d) => ({ key: clean(d.key), options: all(d.options) })),
|
|
818
|
+
})),
|
|
819
|
+
gaps: all(r.gaps),
|
|
820
|
+
...(r.assigned ? { assigned: all(r.assigned) } : {}),
|
|
821
|
+
...(r.prefixes ? { prefixes: r.prefixes.map((n) => ({ ...n, target: clean(n.target), ...(n.why !== undefined ? { why: clean(n.why) } : {}) })) } : {}),
|
|
822
|
+
};
|
|
823
|
+
}
|
|
788
824
|
function lanesCell(l, secrets) {
|
|
789
825
|
if (l.oneLoop)
|
|
790
826
|
return `${l.asked} asked; explored in one loop: ${cell(l.oneLoop, secrets)}`;
|
|
@@ -876,6 +912,8 @@ export function ciSummaryJson(r, version, secrets = []) {
|
|
|
876
912
|
})),
|
|
877
913
|
...(r.capture ? { capture: cleanCapture(r.capture, clean) } : {}),
|
|
878
914
|
...(r.lanes ? { lanes: lanesJson(r.lanes, clean) } : {}),
|
|
915
|
+
...(r.fromRun ? { fromRun: { mode: r.fromRun.mode, source: clean(r.fromRun.source), runId: r.fromRun.runId, recordAt: r.fromRun.recordAt } } : {}),
|
|
916
|
+
...(r.record ? { record: cleanRecord(r.record, clean) } : {}),
|
|
879
917
|
};
|
|
880
918
|
}
|
|
881
919
|
/** The lanes as ci.json holds them. Module paths and lane names come from the app's routes: redacted like the rest. */
|
package/dist/engine/flow.js
CHANGED
|
@@ -205,30 +205,41 @@ const target = z
|
|
|
205
205
|
});
|
|
206
206
|
}
|
|
207
207
|
});
|
|
208
|
+
/**
|
|
209
|
+
* What a step should bring about, in the words of the test it belongs to: the
|
|
210
|
+
* expected result a template-driven test report (`--template`) shows beside
|
|
211
|
+
* the step. Documentation only: it is never checked, so a step passes or
|
|
212
|
+
* fails on its action and its assertions alone.
|
|
213
|
+
*/
|
|
214
|
+
const expected = z.string().trim().min(1, "is the result the step should bring about, in words").max(500).optional();
|
|
208
215
|
/** Most times a repeat step may run its steps. A longer loop is a page that should be reached another way. */
|
|
209
216
|
export const MAX_REPEATS = 100;
|
|
210
217
|
const SINGLE_STEP_SCHEMAS = [
|
|
211
|
-
z.object({ action: z.literal("navigate"), target: z.string().regex(/^\//, "must be a path on the app, starting with /") }).strict(),
|
|
212
|
-
z.object({ action: z.literal("click"), target }).strict(),
|
|
213
|
-
z.object({ action: z.literal("type"), target, value: z.string(), pressEnter: z.boolean().optional(), replace: z.boolean().optional() }).strict(),
|
|
214
|
-
z.object({ action: z.literal("select"), target, value: z.string() }).strict(),
|
|
215
|
-
z.object({ action: z.literal("press"), value: z.string().min(1, "names the key to press, e.g. Enter") }).strict(),
|
|
218
|
+
z.object({ action: z.literal("navigate"), expected, target: z.string().regex(/^\//, "must be a path on the app, starting with /") }).strict(),
|
|
219
|
+
z.object({ action: z.literal("click"), expected, target }).strict(),
|
|
220
|
+
z.object({ action: z.literal("type"), expected, target, value: z.string(), pressEnter: z.boolean().optional(), replace: z.boolean().optional() }).strict(),
|
|
221
|
+
z.object({ action: z.literal("select"), expected, target, value: z.string() }).strict(),
|
|
222
|
+
z.object({ action: z.literal("press"), expected, value: z.string().min(1, "names the key to press, e.g. Enter") }).strict(),
|
|
216
223
|
// A small valid file generated for the input (its kind from `fixture`, else the input's accept attribute), as
|
|
217
224
|
// scout_upload attaches one. `target` is the file input or the control that opens its chooser; absent, the page's only file input.
|
|
218
225
|
z
|
|
219
226
|
.object({
|
|
220
227
|
action: z.literal("upload"),
|
|
228
|
+
expected,
|
|
221
229
|
target: target.optional(),
|
|
222
230
|
fixture: z.enum(FIXTURE_KINDS).optional(),
|
|
223
231
|
name: z.string().min(1).max(120).optional(),
|
|
224
232
|
})
|
|
225
233
|
.strict(),
|
|
226
|
-
z.object({ action: z.literal("expect-text"), text: z.string().min(1, "is the text that must be visible") }).strict(),
|
|
227
|
-
z.object({ action: z.literal("expect-element"), target, state: z.enum(ELEMENT_STATES) }).strict(),
|
|
228
|
-
z
|
|
234
|
+
z.object({ action: z.literal("expect-text"), expected, text: z.string().min(1, "is the text that must be visible") }).strict(),
|
|
235
|
+
z.object({ action: z.literal("expect-element"), expected, target, state: z.enum(ELEMENT_STATES) }).strict(),
|
|
236
|
+
z
|
|
237
|
+
.object({ action: z.literal("expect-url"), expected, pattern: z.string().min(1).refine(isRegex, { message: "is not a valid regular expression" }) })
|
|
238
|
+
.strict(),
|
|
229
239
|
z
|
|
230
240
|
.object({
|
|
231
241
|
action: z.literal("expect-request"),
|
|
242
|
+
expected,
|
|
232
243
|
request: z.string().regex(REQUEST_RE, 'must be a method and a path, e.g. "GET /api/things"'),
|
|
233
244
|
status: z.union([z.number().int().min(100).max(599), z.string().regex(/^[1-5]xx$/, 'must be a status such as 200, or a class such as "2xx"')]),
|
|
234
245
|
})
|
|
@@ -246,6 +257,7 @@ const isRepeatCondition = (s) => s.action === "expect-text" || s.action === "exp
|
|
|
246
257
|
const repeatSchema = z
|
|
247
258
|
.object({
|
|
248
259
|
action: z.literal("repeat"),
|
|
260
|
+
expected,
|
|
249
261
|
steps: z
|
|
250
262
|
.array(singleStepSchema)
|
|
251
263
|
.min(1, "needs at least one step to repeat")
|
|
@@ -262,6 +274,10 @@ const flowSchema = z
|
|
|
262
274
|
.object({
|
|
263
275
|
name: z.string().min(1).max(100).optional(),
|
|
264
276
|
description: z.string().max(500).optional(),
|
|
277
|
+
/** The test's identifier in a template-driven test report (--template). Traceability only: the replay never reads it. */
|
|
278
|
+
id: z.string().trim().min(1, "is the test's identifier").max(100).optional(),
|
|
279
|
+
/** The requirement identifiers the flow covers, shown beside it in a template-driven test report. */
|
|
280
|
+
requirements: z.array(z.string().trim().min(1, "is a requirement identifier").max(100)).max(50, "holds at most 50 identifiers").optional(),
|
|
265
281
|
/**
|
|
266
282
|
* Who walks the flow: a role whose sign-in `scenescout login --role` saved in the project. Absent, the flow runs
|
|
267
283
|
* in the check's own session. Flows run in file-name order, so one role's flow can pick up what another's left.
|
|
@@ -330,6 +346,10 @@ export function parseFlow(text, file) {
|
|
|
330
346
|
const flow = { name: parsed.data.name ?? file.replace(/\.json$/i, ""), file, steps: parsed.data.steps };
|
|
331
347
|
if (parsed.data.role !== undefined)
|
|
332
348
|
flow.role = parsed.data.role;
|
|
349
|
+
if (parsed.data.id !== undefined)
|
|
350
|
+
flow.id = parsed.data.id;
|
|
351
|
+
if (parsed.data.requirements !== undefined)
|
|
352
|
+
flow.requirements = parsed.data.requirements;
|
|
333
353
|
return { ok: true, flow };
|
|
334
354
|
}
|
|
335
355
|
/**
|