scenescout 3.15.0 → 3.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +75 -0
- package/README.md +64 -15
- package/dist/browsers.js +28 -0
- package/dist/check-run.js +190 -14
- package/dist/ci-run.js +268 -52
- package/dist/cli.js +104 -43
- package/dist/commands.js +3 -2
- package/dist/engine/baseline.js +377 -0
- package/dist/engine/brief.js +16 -7
- package/dist/engine/browser.js +1147 -286
- package/dist/engine/calibration.js +61 -30
- package/dist/engine/capture.js +164 -0
- package/dist/engine/check.js +182 -39
- package/dist/engine/ci-lanes.js +215 -0
- package/dist/engine/ci.js +136 -18
- package/dist/engine/claims.js +159 -3
- package/dist/engine/collector.js +561 -30
- package/dist/engine/crawl.js +49 -0
- package/dist/engine/design.js +281 -38
- package/dist/engine/export.js +877 -0
- package/dist/engine/fingerprint.js +92 -4
- package/dist/engine/flow.js +18 -6
- package/dist/engine/forms.js +181 -18
- package/dist/engine/journey.js +29 -1
- package/dist/engine/lane.js +13 -3
- package/dist/engine/launch.js +45 -6
- package/dist/engine/limits.js +7 -0
- package/dist/engine/live-page.js +49 -2
- package/dist/engine/live.js +4 -1
- package/dist/engine/memory.js +501 -47
- package/dist/engine/open.js +118 -0
- package/dist/engine/oracles.js +41 -1
- package/dist/engine/plain.js +268 -0
- package/dist/engine/png.js +127 -0
- package/dist/engine/policy.js +379 -9
- package/dist/engine/probes.js +3 -2
- package/dist/engine/profiles.js +45 -9
- package/dist/engine/project-folder.js +191 -0
- package/dist/engine/refresh.js +68 -3
- package/dist/engine/replay.js +63 -10
- package/dist/engine/report.js +241 -40
- package/dist/engine/request.js +317 -23
- package/dist/engine/sarif.js +120 -0
- package/dist/engine/settle.js +67 -0
- package/dist/engine/signed-in.js +256 -0
- package/dist/engine/tickets.js +671 -0
- package/dist/engine/unload.js +3 -2
- package/dist/export-run.js +633 -0
- package/dist/first-run.js +4 -0
- package/dist/installer.js +267 -3
- package/dist/intake.js +104 -0
- package/dist/login-run.js +250 -36
- package/dist/mcp-server.js +567 -57
- package/dist/playbook.js +5 -0
- package/package.json +6 -3
- package/skills/scenescout/SKILL.md +49 -16
package/dist/engine/unload.js
CHANGED
|
@@ -60,7 +60,8 @@ export function unseenWriteSource(input) {
|
|
|
60
60
|
* interception can know about it. Same rules in the same order: a write
|
|
61
61
|
* another site's frame sends outside the app is refused; so is one sent from
|
|
62
62
|
* a page an embed moved off the app, a sign-in excepted; a sign-in passes;
|
|
63
|
-
* then the
|
|
63
|
+
* then a POST the user named as a read (observe only), then the mode
|
|
64
|
+
* decides. `foreign` and `offApp` come from
|
|
64
65
|
* `unseenWriteSource`, since the sending frame may no longer exist.
|
|
65
66
|
* The one step it does not take is safe-write's short wait for a creation
|
|
66
67
|
* still in flight: a page being left has nothing more to wait for.
|
|
@@ -74,7 +75,7 @@ export function judgeUnseenWrite(input) {
|
|
|
74
75
|
const authExempt = isAuthExempt(mode, method, pathname, destructiveWire);
|
|
75
76
|
if (input.offApp && !authExempt)
|
|
76
77
|
return { allow: false, why: `sent from a page of ${input.offApp}, outside the app` };
|
|
77
|
-
if (authExempt)
|
|
78
|
+
if (authExempt || input.readPost)
|
|
78
79
|
return { allow: true };
|
|
79
80
|
return allowsWrite(mode, method, destructiveWire, input.owned) ? { allow: true } : { allow: false };
|
|
80
81
|
}
|
|
@@ -0,0 +1,633 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Runs `scenescout export`: reads the project's findings, asks the tracker
|
|
3
|
+
* which of them it already holds, and files the rest. The rules (which
|
|
4
|
+
* findings, what an issue says, the marker, the plan) live in
|
|
5
|
+
* engine/export.ts; this file only reads the project's files and talks HTTP.
|
|
6
|
+
* It writes nothing to the project, and the credentials it sends are never
|
|
7
|
+
* printed: every line goes out through a redactor that knows them.
|
|
8
|
+
*/
|
|
9
|
+
import fs from "node:fs";
|
|
10
|
+
import path from "node:path";
|
|
11
|
+
import { redactKeys } from "./engine/ci.js";
|
|
12
|
+
import { credentialSecrets, EXIT_EXPORT, failedCriteriaByFinding, findingIdsInGithubBody, findingIdsInJiraDescription, findingLine, findingPicture, frameIsOriginal, framesFor, githubIssue, jiraEditFields, jiraIssueFields, MARKER_LABEL, MAX_RATE_WAIT_MS, oneLine, planExport, planJiraUpdate, rateLimit, rememberFiled, selectFindings, ticketsToLink, trackerCredentials, trackerMessage, TRACKER_LABEL, uploadName, } from "./engine/export.js";
|
|
13
|
+
import { MEMORY_DIRNAME } from "./engine/memory.js";
|
|
14
|
+
import { backoffMs } from "./engine/provider.js";
|
|
15
|
+
import { resolveFrame } from "./engine/replay.js";
|
|
16
|
+
/**
|
|
17
|
+
* A failure that ends the export. `uncertain`: the tracker may have acted on
|
|
18
|
+
* the request before it failed (a create that timed out or got a 5xx), so
|
|
19
|
+
* whether the issue exists has to be asked again rather than assumed.
|
|
20
|
+
*/
|
|
21
|
+
export class ExportError extends Error {
|
|
22
|
+
uncertain;
|
|
23
|
+
constructor(message, uncertain = false) {
|
|
24
|
+
super(message);
|
|
25
|
+
this.uncertain = uncertain;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
const DEFAULT_TIMEOUT_MS = 20_000;
|
|
29
|
+
const DEFAULT_RETRIES = 3;
|
|
30
|
+
/** Pages of 100 issues read to find what is already filed. More than this and an export refuses rather than file blind. */
|
|
31
|
+
const MAX_PAGES = 50;
|
|
32
|
+
/** A frame larger than this is not uploaded: a recorded frame is a JPEG of a page, far smaller. */
|
|
33
|
+
const MAX_FRAME_BYTES = 10_000_000;
|
|
34
|
+
async function readBody(res) {
|
|
35
|
+
const text = await res.text();
|
|
36
|
+
if (!text)
|
|
37
|
+
return null;
|
|
38
|
+
try {
|
|
39
|
+
return JSON.parse(text);
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
// Not JSON: an HTML error page from a proxy, say. The text is still what the message quotes.
|
|
43
|
+
return { message: text.slice(0, 300) };
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
const REFUSAL_HINT = {
|
|
47
|
+
400: " (a field was refused: in Jira, check --jira-issue-type and the --severity-map priorities)",
|
|
48
|
+
401: " (the credentials were not accepted)",
|
|
49
|
+
403: " (the credentials lack a permission this needs)",
|
|
50
|
+
404: " (no such repository or project, or the credentials cannot see it)",
|
|
51
|
+
410: " (issues are turned off for this repository)",
|
|
52
|
+
};
|
|
53
|
+
/** Why a request got no usable answer, as one line: Node's fetch says only "fetch failed" and keeps the reason in `cause`. */
|
|
54
|
+
function failureOf(err, timeoutMs) {
|
|
55
|
+
if (err instanceof Error && err.name === "TimeoutError")
|
|
56
|
+
return `no answer within ${timeoutMs / 1000} s`;
|
|
57
|
+
if (!(err instanceof Error))
|
|
58
|
+
return oneLine(err, 200);
|
|
59
|
+
const cause = err.cause instanceof Error ? err.cause : null;
|
|
60
|
+
const code = cause && typeof cause.code === "string" ? `${cause.code} ` : "";
|
|
61
|
+
return oneLine(cause ? `${err.message}: ${code}${cause.message}` : err.message, 300);
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* One tracker's requests: its base address, its credentials, a timeout on
|
|
65
|
+
* every request (reading the answer included), no redirect followed (the
|
|
66
|
+
* credentials would go with it), a rate limit waited out when it asks for a
|
|
67
|
+
* minute or less, and a 5xx, a dropped connection or an answer cut off
|
|
68
|
+
* retried with backoff when the request is safe to send twice. When it is
|
|
69
|
+
* not, the failure is marked uncertain: the tracker may have acted on it.
|
|
70
|
+
*/
|
|
71
|
+
function client(base, headers, h) {
|
|
72
|
+
return async (r) => {
|
|
73
|
+
const what = `${r.method} ${r.path.split("?")[0]}`;
|
|
74
|
+
for (let attempt = 0;; attempt++) {
|
|
75
|
+
let res;
|
|
76
|
+
let body;
|
|
77
|
+
try {
|
|
78
|
+
res = await h.fetchImpl(`${base}${r.path}`, {
|
|
79
|
+
method: r.method,
|
|
80
|
+
headers: { accept: "application/json", ...headers, ...(r.json !== undefined ? { "content-type": "application/json" } : {}), ...r.headers },
|
|
81
|
+
body: r.json !== undefined ? JSON.stringify(r.json) : r.form,
|
|
82
|
+
redirect: "manual",
|
|
83
|
+
signal: AbortSignal.timeout(h.timeoutMs),
|
|
84
|
+
});
|
|
85
|
+
body = await readBody(res);
|
|
86
|
+
}
|
|
87
|
+
catch (err) {
|
|
88
|
+
if (r.idempotent && attempt < h.retries) {
|
|
89
|
+
await h.wait(backoffMs(attempt + 1, h.random));
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
92
|
+
throw new ExportError(`${what} failed: ${failureOf(err, h.timeoutMs)}`, !r.idempotent);
|
|
93
|
+
}
|
|
94
|
+
if (res.status >= 300 && res.status < 400)
|
|
95
|
+
throw new ExportError(`${what} was answered with a redirect (HTTP ${res.status}), which is refused: credentials are never sent on to another address. A repository or site that moved needs its new address`);
|
|
96
|
+
const said = trackerMessage(body);
|
|
97
|
+
const limit = rateLimit(res.status, (name) => res.headers.get(name), h.now(), said);
|
|
98
|
+
if (limit.limited) {
|
|
99
|
+
const waitMs = limit.waitMs ?? backoffMs(attempt + 1, h.random);
|
|
100
|
+
if (waitMs > MAX_RATE_WAIT_MS)
|
|
101
|
+
throw new ExportError(`${what} hit the tracker's rate limit, which asks to wait ${Math.ceil(waitMs / 1000)} s; export again after that`);
|
|
102
|
+
if (attempt >= h.retries)
|
|
103
|
+
throw new ExportError(`${what} hit the tracker's rate limit ${attempt + 1} times; export again later`);
|
|
104
|
+
await h.wait(waitMs);
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
if (res.status >= 500) {
|
|
108
|
+
if (r.idempotent && attempt < h.retries) {
|
|
109
|
+
await h.wait(backoffMs(attempt + 1, h.random));
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
throw new ExportError(`${what} failed: HTTP ${res.status}${said ? `: ${said}` : ""}`, !r.idempotent);
|
|
113
|
+
}
|
|
114
|
+
if (!res.ok)
|
|
115
|
+
throw new ExportError(`${what} was refused: HTTP ${res.status}${REFUSAL_HINT[res.status] ?? ""}${said ? `: ${said}` : ""}`);
|
|
116
|
+
return body;
|
|
117
|
+
}
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
const isRecord = (v) => !!v && typeof v === "object" && !Array.isArray(v);
|
|
121
|
+
function githubApi({ repo }, send) {
|
|
122
|
+
const root = `/repos/${repo
|
|
123
|
+
.split("/")
|
|
124
|
+
.map((p) => encodeURIComponent(p))
|
|
125
|
+
.join("/")}/issues`;
|
|
126
|
+
return {
|
|
127
|
+
label: `GitHub ${repo}`,
|
|
128
|
+
consistentListing: true,
|
|
129
|
+
// Listed by label rather than found through the search API: search is
|
|
130
|
+
// indexed with a delay, so an export run right after another would not
|
|
131
|
+
// see the issues it had just filed and would file them again.
|
|
132
|
+
async existing(includeClosed) {
|
|
133
|
+
const map = new Map();
|
|
134
|
+
let issues = 0;
|
|
135
|
+
let unmarked = 0;
|
|
136
|
+
for (let page = 1; page <= MAX_PAGES; page++) {
|
|
137
|
+
const body = await send({
|
|
138
|
+
method: "GET",
|
|
139
|
+
path: `${root}?labels=${encodeURIComponent(MARKER_LABEL)}&state=${includeClosed ? "all" : "open"}&per_page=100&page=${page}`,
|
|
140
|
+
idempotent: true,
|
|
141
|
+
});
|
|
142
|
+
if (!Array.isArray(body))
|
|
143
|
+
throw new ExportError("GitHub's list of issues was not a list");
|
|
144
|
+
for (const issue of body) {
|
|
145
|
+
// The issues API lists pull requests too.
|
|
146
|
+
if (isRecord(issue) && issue.pull_request)
|
|
147
|
+
continue;
|
|
148
|
+
const number = isRecord(issue) ? Number(issue.number) : NaN;
|
|
149
|
+
// An entry that cannot be read, or one without its body, could be a filed finding: guessing "not filed" files it twice.
|
|
150
|
+
if (!isRecord(issue) || !Number.isInteger(number) || !("body" in issue))
|
|
151
|
+
throw new ExportError("GitHub listed an issue without its number or its body, so which findings are filed cannot be told");
|
|
152
|
+
issues++;
|
|
153
|
+
const ids = findingIdsInGithubBody(issue.body);
|
|
154
|
+
if (ids.length === 0)
|
|
155
|
+
unmarked++;
|
|
156
|
+
for (const id of ids)
|
|
157
|
+
rememberFiled(map, id, { ref: `#${number}`, number, url: oneLine(issue.html_url, 300), open: issue.state !== "closed" });
|
|
158
|
+
}
|
|
159
|
+
if (body.length < 100)
|
|
160
|
+
return { map, issues, unmarked };
|
|
161
|
+
}
|
|
162
|
+
throw new ExportError(`${MAX_PAGES * 100} or more issues carry the ${MARKER_LABEL} label, too many to check them all for this export's findings`);
|
|
163
|
+
},
|
|
164
|
+
async create(f, ctx) {
|
|
165
|
+
const body = await send({ method: "POST", path: root, json: githubIssue(f, ctx), idempotent: false });
|
|
166
|
+
const number = isRecord(body) ? Number(body.number) : NaN;
|
|
167
|
+
if (!isRecord(body) || !Number.isInteger(number))
|
|
168
|
+
throw new ExportError("GitHub answered the create without an issue number", true);
|
|
169
|
+
const issue = { ref: `#${number}`, number, url: oneLine(body.html_url, 300), open: true };
|
|
170
|
+
const labels = Array.isArray(body.labels) ? body.labels : [];
|
|
171
|
+
const names = labels.map((l) => (typeof l === "string" ? l : isRecord(l) ? String(l.name ?? "") : "").toLowerCase());
|
|
172
|
+
// GitHub drops the labels of an issue created by an account that cannot set them, and says nothing.
|
|
173
|
+
if (!names.includes(MARKER_LABEL))
|
|
174
|
+
return {
|
|
175
|
+
issue,
|
|
176
|
+
problems: [],
|
|
177
|
+
stop: `GitHub filed ${issue.ref} without its ${MARKER_LABEL} label, so a later export could not find it. Stopping before filing more: GitHub drops the labels of an issue created by an account that may not set them, so give the token's account write access to the repository (and create the ${MARKER_LABEL} label if it has none), then add the label to ${issue.ref} by hand`,
|
|
178
|
+
};
|
|
179
|
+
return { issue, problems: [] };
|
|
180
|
+
},
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
function jiraApi({ baseUrl, projectKey, issueType, linkType }, send) {
|
|
184
|
+
const keyNumber = (key) => {
|
|
185
|
+
const m = /^[A-Z][A-Z0-9_]*-(\d+)$/.exec(key);
|
|
186
|
+
return m ? Number(m[1]) : null;
|
|
187
|
+
};
|
|
188
|
+
const issuePath = (key) => `/rest/api/3/issue/${encodeURIComponent(key)}`;
|
|
189
|
+
/** Attach each file; what could not be attached is returned, not thrown, since the issue itself stands. */
|
|
190
|
+
const attach = async (key, files, again) => {
|
|
191
|
+
const done = [];
|
|
192
|
+
const problems = [];
|
|
193
|
+
for (const frame of files) {
|
|
194
|
+
const name = frame.upload;
|
|
195
|
+
try {
|
|
196
|
+
const form = new FormData();
|
|
197
|
+
form.append("file", new Blob([fs.readFileSync(frame.file)], { type: name.endsWith(".png") ? "image/png" : "image/jpeg" }), name);
|
|
198
|
+
await send({
|
|
199
|
+
method: "POST",
|
|
200
|
+
path: `${issuePath(key)}/attachments`,
|
|
201
|
+
form,
|
|
202
|
+
// Jira refuses an upload without it, as protection against cross-site requests.
|
|
203
|
+
headers: { "x-atlassian-token": "no-check" },
|
|
204
|
+
idempotent: true,
|
|
205
|
+
});
|
|
206
|
+
done.push(name);
|
|
207
|
+
}
|
|
208
|
+
catch (err) {
|
|
209
|
+
problems.push(`${name} was not attached (${err instanceof Error ? err.message : String(err)}). It is at ${frame.file}; ${again}`);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
return { done, problems };
|
|
213
|
+
};
|
|
214
|
+
/**
|
|
215
|
+
* Link the issue to each ticket whose criterion it fails, so the issue reads
|
|
216
|
+
* as the subject of the link type's outward words ("blocks", "relates to")
|
|
217
|
+
* and the ticket as their object. Jira's API gives those words to the issue
|
|
218
|
+
* sent as `inwardIssue`, the reverse of what the field names suggest. Jira
|
|
219
|
+
* answers a link that already exists as made, so sending one twice is harmless.
|
|
220
|
+
*/
|
|
221
|
+
const link = async (key, tickets) => {
|
|
222
|
+
const done = [];
|
|
223
|
+
const problems = [];
|
|
224
|
+
if (!linkType)
|
|
225
|
+
return { done, problems };
|
|
226
|
+
for (const ticket of tickets) {
|
|
227
|
+
try {
|
|
228
|
+
await send({
|
|
229
|
+
method: "POST",
|
|
230
|
+
path: "/rest/api/3/issueLink",
|
|
231
|
+
json: { type: { name: linkType }, inwardIssue: { key }, outwardIssue: { key: ticket } },
|
|
232
|
+
idempotent: true,
|
|
233
|
+
});
|
|
234
|
+
done.push(ticket);
|
|
235
|
+
}
|
|
236
|
+
catch (err) {
|
|
237
|
+
problems.push(`not linked to ${ticket} (${err instanceof Error ? err.message : String(err)}); check that ${ticket} is an issue on this site and that "${linkType}" is a link type it has (--jira-link-type), and a later export links it`);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
return { done, problems };
|
|
241
|
+
};
|
|
242
|
+
const keysOf = (links) => (Array.isArray(links) ? links : []).flatMap((l) => isRecord(l) ? [l.inwardIssue, l.outwardIssue].flatMap((i) => (isRecord(i) && typeof i.key === "string" ? [i.key] : [])) : []);
|
|
243
|
+
const namesOf = (attachments) => (Array.isArray(attachments) ? attachments : []).flatMap((a) => (isRecord(a) && typeof a.filename === "string" ? [a.filename] : []));
|
|
244
|
+
return {
|
|
245
|
+
label: `Jira ${projectKey}`,
|
|
246
|
+
consistentListing: false,
|
|
247
|
+
async existing(includeClosed) {
|
|
248
|
+
// The key was checked to be capitals, digits and underscores; the label is ours. Nothing else goes into the query.
|
|
249
|
+
const jql = `project = "${projectKey}" AND labels = "${MARKER_LABEL}"${includeClosed ? "" : " AND statusCategory != Done"} ORDER BY created ASC`;
|
|
250
|
+
const map = new Map();
|
|
251
|
+
let issues = 0;
|
|
252
|
+
let unmarked = 0;
|
|
253
|
+
let nextPageToken;
|
|
254
|
+
for (let page = 1; page <= MAX_PAGES; page++) {
|
|
255
|
+
const body = await send({
|
|
256
|
+
method: "POST",
|
|
257
|
+
path: "/rest/api/3/search/jql",
|
|
258
|
+
json: { jql, fields: ["summary", "description", "status", "attachment", "issuelinks"], maxResults: 100, ...(nextPageToken ? { nextPageToken } : {}) },
|
|
259
|
+
idempotent: true,
|
|
260
|
+
});
|
|
261
|
+
if (!isRecord(body) || !Array.isArray(body.issues))
|
|
262
|
+
throw new ExportError("Jira's search answered without a list of issues");
|
|
263
|
+
for (const issue of body.issues) {
|
|
264
|
+
const key = isRecord(issue) ? String(issue.key ?? "") : "";
|
|
265
|
+
const number = keyNumber(key);
|
|
266
|
+
const fields = isRecord(issue) && isRecord(issue.fields) ? issue.fields : null;
|
|
267
|
+
// A field configuration can hide the description; without it, every finding would read as unfiled and be filed again.
|
|
268
|
+
if (number === null || !fields || !("description" in fields))
|
|
269
|
+
throw new ExportError(`Jira listed an issue without its key or its description, so which findings are filed cannot be told`);
|
|
270
|
+
issues++;
|
|
271
|
+
const status = isRecord(fields.status) && isRecord(fields.status.statusCategory) ? fields.status.statusCategory.key : undefined;
|
|
272
|
+
const ids = findingIdsInJiraDescription(fields.description);
|
|
273
|
+
if (ids.length === 0)
|
|
274
|
+
unmarked++;
|
|
275
|
+
const jira = { summary: fields.summary, description: fields.description, attachments: namesOf(fields.attachment), links: keysOf(fields.issuelinks) };
|
|
276
|
+
for (const id of ids)
|
|
277
|
+
rememberFiled(map, id, { ref: key, number, url: `${baseUrl}/browse/${key}`, open: status !== "done", jira });
|
|
278
|
+
}
|
|
279
|
+
nextPageToken = typeof body.nextPageToken === "string" && body.nextPageToken ? body.nextPageToken : undefined;
|
|
280
|
+
if (!nextPageToken || body.isLast === true)
|
|
281
|
+
return { map, issues, unmarked };
|
|
282
|
+
}
|
|
283
|
+
throw new ExportError(`more than ${MAX_PAGES} pages of issues in ${projectKey} carry the ${MARKER_LABEL} label, too many to check them all for this export's findings`);
|
|
284
|
+
},
|
|
285
|
+
async create(f, ctx, extras) {
|
|
286
|
+
const body = await send({
|
|
287
|
+
method: "POST",
|
|
288
|
+
path: "/rest/api/3/issue",
|
|
289
|
+
json: { fields: jiraIssueFields(f, { ...ctx, projectKey, issueType }) },
|
|
290
|
+
idempotent: false,
|
|
291
|
+
});
|
|
292
|
+
const key = isRecord(body) ? String(body.key ?? "") : "";
|
|
293
|
+
const number = keyNumber(key);
|
|
294
|
+
if (number === null)
|
|
295
|
+
throw new ExportError("Jira answered the create without an issue key", true);
|
|
296
|
+
const issue = { ref: key, number, url: `${baseUrl}/browse/${key}`, open: true };
|
|
297
|
+
const attached = await attach(key, extras.files, "a later export attaches it while the issue is open, unless --jira-update is off");
|
|
298
|
+
const linked = await link(key, extras.tickets);
|
|
299
|
+
return { issue, problems: [...attached.problems, ...linked.problems] };
|
|
300
|
+
},
|
|
301
|
+
async update(f, ctx, issue, plan, extras) {
|
|
302
|
+
const rewrote = plan.fields === "change";
|
|
303
|
+
if (rewrote)
|
|
304
|
+
// Summary and description only: an edit, so Jira notifies the issue's watchers as it does for any other.
|
|
305
|
+
await send({ method: "PUT", path: issuePath(issue.ref), json: { fields: jiraEditFields(f, ctx) }, idempotent: true });
|
|
306
|
+
const wanted = new Set(plan.attach);
|
|
307
|
+
const attached = await attach(issue.ref, extras.files.filter((x) => wanted.has(x.upload)), "a later export tries again");
|
|
308
|
+
const linked = await link(issue.ref, plan.link);
|
|
309
|
+
return { rewrote, attached: attached.done, linked: linked.done, problems: [...attached.problems, ...linked.problems] };
|
|
310
|
+
},
|
|
311
|
+
};
|
|
312
|
+
}
|
|
313
|
+
// ── the project's files ─────────────────────────────────────────────────────
|
|
314
|
+
/** The project's memory, refused unless it is one this version reads: a file it cannot read exported nothing and said so with exit 0. */
|
|
315
|
+
function readMemory(memoryPath) {
|
|
316
|
+
if (!fs.existsSync(memoryPath))
|
|
317
|
+
throw new ExportError(`no findings to export: ${memoryPath} does not exist. Run SceneScout on this project first, or pass --project`);
|
|
318
|
+
let memory;
|
|
319
|
+
try {
|
|
320
|
+
memory = JSON.parse(fs.readFileSync(memoryPath, "utf8"));
|
|
321
|
+
}
|
|
322
|
+
catch (err) {
|
|
323
|
+
throw new ExportError(`could not read ${memoryPath}: ${err instanceof Error ? err.message : String(err)}`);
|
|
324
|
+
}
|
|
325
|
+
if (!isRecord(memory) || memory.version !== 1)
|
|
326
|
+
throw new ExportError(`${memoryPath} is not a memory file this version reads (version ${isRecord(memory) ? oneLine(memory.version, 20) : "unknown"})`);
|
|
327
|
+
if (!Array.isArray(memory.findings))
|
|
328
|
+
throw new ExportError(`${memoryPath} holds no list of findings`);
|
|
329
|
+
return memory;
|
|
330
|
+
}
|
|
331
|
+
/** Every step with a frame, from every session log in the memory directory. Logs and lines that cannot be read are counted, not fatal. */
|
|
332
|
+
function framedSteps(memoryDir) {
|
|
333
|
+
const steps = [];
|
|
334
|
+
let unreadable = 0;
|
|
335
|
+
if (!fs.existsSync(path.join(memoryDir, "recordings")))
|
|
336
|
+
return { steps, unreadable };
|
|
337
|
+
for (const name of fs.readdirSync(memoryDir)) {
|
|
338
|
+
if (!/^session-.*\.jsonl$/.test(name))
|
|
339
|
+
continue;
|
|
340
|
+
let text;
|
|
341
|
+
try {
|
|
342
|
+
text = fs.readFileSync(path.join(memoryDir, name), "utf8");
|
|
343
|
+
}
|
|
344
|
+
catch {
|
|
345
|
+
// A log that cannot be read costs only its screenshots; it is counted with the lines that do not parse.
|
|
346
|
+
unreadable++;
|
|
347
|
+
continue;
|
|
348
|
+
}
|
|
349
|
+
for (const line of text.split("\n")) {
|
|
350
|
+
if (!line.includes('"frame"'))
|
|
351
|
+
continue;
|
|
352
|
+
try {
|
|
353
|
+
const e = JSON.parse(line);
|
|
354
|
+
if (typeof e.at === "string" && typeof e.frame === "string")
|
|
355
|
+
steps.push({ at: e.at, session: e.session, frame: e.frame });
|
|
356
|
+
}
|
|
357
|
+
catch {
|
|
358
|
+
unreadable++;
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
return { steps, unreadable };
|
|
363
|
+
}
|
|
364
|
+
/** A frame's file, when it really is inside the recordings folder: links are followed first, so a link to elsewhere, or a linked folder, is not. */
|
|
365
|
+
function frameFile(root, rel) {
|
|
366
|
+
const named = resolveFrame(root, rel);
|
|
367
|
+
if (!named)
|
|
368
|
+
return null;
|
|
369
|
+
try {
|
|
370
|
+
const realRoot = fs.realpathSync(root);
|
|
371
|
+
const file = fs.realpathSync(named);
|
|
372
|
+
if (!file.startsWith(realRoot + path.sep))
|
|
373
|
+
return null;
|
|
374
|
+
const stat = fs.statSync(file);
|
|
375
|
+
return stat.isFile() ? { file, stat } : null;
|
|
376
|
+
}
|
|
377
|
+
catch {
|
|
378
|
+
// Missing, or unreadable: this frame is left out and counted, like any other that cannot be attached.
|
|
379
|
+
return null;
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
/** The frames an issue names or attaches: inside the recordings folder, small enough, and still the picture of that step. */
|
|
383
|
+
function framesOf(f, steps, memoryDir) {
|
|
384
|
+
const frames = [];
|
|
385
|
+
let dropped = 0;
|
|
386
|
+
for (const { frame, at } of framesFor(f, steps)) {
|
|
387
|
+
const found = frameFile(path.join(memoryDir, "recordings"), frame);
|
|
388
|
+
if (!found || found.stat.size > MAX_FRAME_BYTES || !frameIsOriginal(at, found.stat.mtimeMs)) {
|
|
389
|
+
dropped++;
|
|
390
|
+
continue;
|
|
391
|
+
}
|
|
392
|
+
frames.push({ rel: frame, file: found.file, at, upload: uploadName(frame, at) });
|
|
393
|
+
}
|
|
394
|
+
return { frames, dropped };
|
|
395
|
+
}
|
|
396
|
+
/** The finding's own picture, when its path is one the engine writes and the file is inside the recordings folder and small enough. */
|
|
397
|
+
function pictureOf(f, memoryDir) {
|
|
398
|
+
const picture = findingPicture(f);
|
|
399
|
+
if (!picture)
|
|
400
|
+
return null;
|
|
401
|
+
const found = frameFile(path.join(memoryDir, "recordings"), picture.rel);
|
|
402
|
+
return found && found.stat.size <= MAX_FRAME_BYTES ? { ...picture, file: found.file, upload: uploadName(picture.rel, picture.at) } : null;
|
|
403
|
+
}
|
|
404
|
+
// ── the export ──────────────────────────────────────────────────────────────
|
|
405
|
+
const OUTCOME_WORDS = { file: "would file", "already-filed": "already filed", "over-cap": "over the cap" };
|
|
406
|
+
export async function runExport(o, deps) {
|
|
407
|
+
const credentials = trackerCredentials(o.to, deps.env);
|
|
408
|
+
// Every credential that is set, not only the tracker's: one echoed back from anywhere is never printed.
|
|
409
|
+
const secrets = credentialSecrets(deps.env);
|
|
410
|
+
const say = (line) => deps.log(redactKeys(line, secrets));
|
|
411
|
+
const complain = (line) => (deps.error ?? deps.log)(redactKeys(line, secrets));
|
|
412
|
+
const http = {
|
|
413
|
+
fetchImpl: deps.fetchImpl ?? fetch,
|
|
414
|
+
wait: deps.wait ?? ((ms) => new Promise((r) => setTimeout(r, ms))),
|
|
415
|
+
timeoutMs: deps.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
|
416
|
+
retries: deps.retries ?? DEFAULT_RETRIES,
|
|
417
|
+
now: deps.now ?? Date.now,
|
|
418
|
+
random: deps.random ?? Math.random,
|
|
419
|
+
};
|
|
420
|
+
const pauseMs = deps.pauseMs ?? (o.to === "github" ? 1000 : 250);
|
|
421
|
+
const where = o.to === "github" ? `GitHub ${o.github.repo}` : `Jira ${o.jira.projectKey} at ${o.jira.baseUrl}`;
|
|
422
|
+
const filed = [];
|
|
423
|
+
const updated = [];
|
|
424
|
+
let plan = [];
|
|
425
|
+
// Kept outside the try, so an export that stops part-way still says what went wrong before it stopped.
|
|
426
|
+
const problems = [];
|
|
427
|
+
let droppedFrames = 0;
|
|
428
|
+
const reportFrames = () => {
|
|
429
|
+
if (droppedFrames > 0)
|
|
430
|
+
say(`Left out ${droppedFrames} frame(s) that are missing, too large, or were rewritten by a later run.`);
|
|
431
|
+
};
|
|
432
|
+
try {
|
|
433
|
+
const memoryDir = path.join(o.projectDir, MEMORY_DIRNAME);
|
|
434
|
+
const memoryPath = path.join(memoryDir, "memory.json");
|
|
435
|
+
const memory = readMemory(memoryPath);
|
|
436
|
+
const selection = selectFindings(memory, o);
|
|
437
|
+
const criteria = failedCriteriaByFinding(memory);
|
|
438
|
+
if (selection.unknownOnly.length > 0)
|
|
439
|
+
throw new ExportError(`--only names no finding of this project: ${selection.unknownOnly.join(", ")}`);
|
|
440
|
+
say(o.dryRun ? `SceneScout export to ${where}: a dry run, so nothing is filed. Pass --yes to file.` : `SceneScout export to ${where}.`);
|
|
441
|
+
const c = selection.counts;
|
|
442
|
+
const left = [
|
|
443
|
+
c.belowSeverity ? `${c.belowSeverity} below --min-severity ${o.minSeverity}` : "",
|
|
444
|
+
c.worthALook ? `${c.worthALook} worth a look (--include-worth-a-look exports them)` : "",
|
|
445
|
+
].filter(Boolean);
|
|
446
|
+
say(`Findings in ${memoryPath}: ${c.open} open, ${c.resolved} resolved${selection.unreadable.length ? `, ${selection.unreadable.length} unreadable` : ""}. ` +
|
|
447
|
+
`${selection.candidates.length} to export${left.length ? `; left out: ${left.join(", ")}` : ""}.`);
|
|
448
|
+
if (selection.unreadable.length > 0)
|
|
449
|
+
say(` Not exported, as this version cannot read them: ${selection.unreadable.slice(0, 20).join(", ")}${selection.unreadable.length > 20 ? " …" : ""}`);
|
|
450
|
+
for (const { id, reason } of selection.leftOut)
|
|
451
|
+
say(` not exported ${id}: ${reason}`);
|
|
452
|
+
if (selection.candidates.length === 0) {
|
|
453
|
+
say("Nothing to export.");
|
|
454
|
+
return { exitCode: EXIT_EXPORT.done, plan, filed, updated };
|
|
455
|
+
}
|
|
456
|
+
if (!credentials.ok && !o.dryRun)
|
|
457
|
+
throw new ExportError(credentials.error);
|
|
458
|
+
const tracker = !credentials.ok
|
|
459
|
+
? null
|
|
460
|
+
: o.to === "github"
|
|
461
|
+
? githubApi(o.github, client(o.github.apiUrl, { ...credentials.headers, ...GITHUB_HEADERS }, http))
|
|
462
|
+
: jiraApi(o.jira, client(o.jira.baseUrl, credentials.headers, http));
|
|
463
|
+
// A closed issue counts as filed unless --refile-closed.
|
|
464
|
+
const includeClosed = !o.refileClosed;
|
|
465
|
+
let existing = null;
|
|
466
|
+
if (tracker) {
|
|
467
|
+
const found = await tracker.existing(includeClosed);
|
|
468
|
+
existing = found.map;
|
|
469
|
+
say(`Compared with ${tracker.label}: ${found.issues} ${includeClosed ? "" : "open "}issue(s) carry the ${MARKER_LABEL} label` +
|
|
470
|
+
`${includeClosed ? " (open or closed)" : ""}${found.unmarked ? `, ${found.unmarked} of them with no marker` : ""}` +
|
|
471
|
+
`${includeClosed ? "" : "; a finding whose issue was closed is filed again (--refile-closed)"}.`);
|
|
472
|
+
}
|
|
473
|
+
else if (!credentials.ok) {
|
|
474
|
+
say(`Not compared with ${TRACKER_LABEL[o.to]}: ${credentials.error}. A finding filed earlier is listed below as one to file.`);
|
|
475
|
+
}
|
|
476
|
+
plan = planExport(selection.candidates, existing, o.maxIssues);
|
|
477
|
+
const steps = o.screenshots ? framedSteps(memoryDir) : { steps: [], unreadable: 0 };
|
|
478
|
+
if (steps.unreadable > 0)
|
|
479
|
+
say(`Skipped ${steps.unreadable} session-log line(s) or file(s) that could not be read while looking for screenshots.`);
|
|
480
|
+
const contextOf = (f) => {
|
|
481
|
+
const found = o.screenshots ? framesOf(f, steps.steps, memoryDir) : { frames: [], dropped: 0 };
|
|
482
|
+
droppedFrames += found.dropped;
|
|
483
|
+
const picture = o.screenshots ? pictureOf(f, memoryDir) : null;
|
|
484
|
+
const failed = criteria.get(f.id) ?? [];
|
|
485
|
+
return {
|
|
486
|
+
ctx: {
|
|
487
|
+
severityName: o.severityMap[f.severity],
|
|
488
|
+
labels: o.labels,
|
|
489
|
+
screenshots: o.screenshots ? found.frames.map((x) => x.rel) : "off",
|
|
490
|
+
framesLeftOut: found.dropped,
|
|
491
|
+
picture: picture?.rel ?? null,
|
|
492
|
+
...(picture?.at ? { pictureAt: picture.at } : {}),
|
|
493
|
+
screenshotTimes: found.frames.map((x) => x.at ?? ""),
|
|
494
|
+
criteria: failed,
|
|
495
|
+
},
|
|
496
|
+
// GitHub takes no uploads and has no links between trackers: its body names both instead.
|
|
497
|
+
extras: o.to === "jira"
|
|
498
|
+
? { files: [...(picture ? [picture] : []), ...found.frames], tickets: o.jira.linkType ? ticketsToLink(failed) : [] }
|
|
499
|
+
: { files: [], tickets: [] },
|
|
500
|
+
};
|
|
501
|
+
};
|
|
502
|
+
let writes = 0;
|
|
503
|
+
for (const entry of plan) {
|
|
504
|
+
const f = entry.finding;
|
|
505
|
+
if (entry.outcome === "already-filed") {
|
|
506
|
+
const issue = entry.issue;
|
|
507
|
+
const already = ` ${OUTCOME_WORDS[entry.outcome].padEnd(13)} ${findingLine(f)} as ${issue.ref}${issue.open ? "" : " (closed)"}`;
|
|
508
|
+
// Only an open Jira issue filed earlier is brought up to date; a closed one is the team's decision and left alone.
|
|
509
|
+
if (o.to !== "jira" || !o.jira.update || !issue.open || !issue.jira || !tracker?.update) {
|
|
510
|
+
say(already);
|
|
511
|
+
continue;
|
|
512
|
+
}
|
|
513
|
+
const { ctx, extras } = contextOf(f);
|
|
514
|
+
const wanted = jiraEditFields(f, ctx);
|
|
515
|
+
const upd = planJiraUpdate(f, issue.jira, {
|
|
516
|
+
...wanted,
|
|
517
|
+
files: extras.files.map((x) => x.upload),
|
|
518
|
+
tickets: extras.tickets,
|
|
519
|
+
});
|
|
520
|
+
const changesOf = (rewrite, attach, link, tense) => [
|
|
521
|
+
rewrite ? (tense === "would" ? "rewrite its summary and description" : "rewrote its summary and description") : "",
|
|
522
|
+
attach.length ? `${tense === "would" ? "attach" : "attached"} ${attach.join(", ")}` : "",
|
|
523
|
+
link.length ? `${tense === "would" ? "link it to" : "linked it to"} ${link.join(", ")}` : "",
|
|
524
|
+
].filter(Boolean);
|
|
525
|
+
const changes = changesOf(upd.fields === "change", upd.attach, upd.link, "would");
|
|
526
|
+
const kept = upd.fields === "edited"
|
|
527
|
+
? "; its summary or description was edited in Jira, so it is left as written"
|
|
528
|
+
: upd.fields === "no-revision"
|
|
529
|
+
? "; it was filed by an earlier version, so its summary and description are left as they are"
|
|
530
|
+
: "";
|
|
531
|
+
if (changes.length === 0 || o.dryRun) {
|
|
532
|
+
const would = changes.length ? `; would ${changes.join(", ")}` : upd.fields === "same" ? ", up to date" : "";
|
|
533
|
+
say(`${already}${would}${kept}`);
|
|
534
|
+
continue;
|
|
535
|
+
}
|
|
536
|
+
if (writes > 0 && pauseMs > 0)
|
|
537
|
+
await http.wait(pauseMs);
|
|
538
|
+
writes++;
|
|
539
|
+
const did = await tracker.update(f, ctx, issue, upd, extras);
|
|
540
|
+
const done = changesOf(did.rewrote, did.attached, did.linked, "did");
|
|
541
|
+
// Said only of what Jira took: an update whose every attachment and link failed changed nothing.
|
|
542
|
+
if (done.length > 0) {
|
|
543
|
+
updated.push({ id: f.id, issue });
|
|
544
|
+
say(` ${`updated ${issue.ref}`.padEnd(13)} ${findingLine(f)} ${issue.url}: ${done.join(", ")}${kept}`);
|
|
545
|
+
}
|
|
546
|
+
else
|
|
547
|
+
say(`${already}; nothing could be updated${kept}`);
|
|
548
|
+
for (const p of did.problems)
|
|
549
|
+
problems.push(`${issue.ref}: ${p}`);
|
|
550
|
+
continue;
|
|
551
|
+
}
|
|
552
|
+
// Listed, not filed: over the cap, or a dry run (the only way to be here with no tracker, as --yes needs credentials).
|
|
553
|
+
if (entry.outcome === "over-cap" || o.dryRun || !tracker) {
|
|
554
|
+
const shown = entry.outcome === "file" && o.screenshots ? contextOf(f).ctx : null;
|
|
555
|
+
const shots = shown && shown.screenshots !== "off" ? shown.screenshots.length + (shown.picture ? 1 : 0) : 0;
|
|
556
|
+
say(` ${OUTCOME_WORDS[entry.outcome].padEnd(13)} ${findingLine(f)}${shots ? ` with ${shots} screenshot(s)` : ""}`);
|
|
557
|
+
continue;
|
|
558
|
+
}
|
|
559
|
+
if (writes > 0 && pauseMs > 0)
|
|
560
|
+
await http.wait(pauseMs);
|
|
561
|
+
const { ctx, extras } = contextOf(f);
|
|
562
|
+
const done = await fileOne(tracker, f, ctx, extras, includeClosed, http);
|
|
563
|
+
writes++;
|
|
564
|
+
filed.push({ id: f.id, issue: done.issue });
|
|
565
|
+
say(` ${`filed ${done.issue.ref}`.padEnd(13)} ${findingLine(f)} ${done.issue.url}${done.confirmed ? " (found by its marker after the tracker's error)" : ""}`);
|
|
566
|
+
for (const p of done.problems)
|
|
567
|
+
problems.push(`${done.issue.ref}: ${p}`);
|
|
568
|
+
if (done.stop)
|
|
569
|
+
throw new ExportError(done.stop);
|
|
570
|
+
}
|
|
571
|
+
reportFrames();
|
|
572
|
+
const count = (outcome) => plan.filter((e) => e.outcome === outcome).length;
|
|
573
|
+
const over = count("over-cap");
|
|
574
|
+
const overText = over ? `; ${over} over the cap of ${o.maxIssues} (--max-issues): export again to file them` : "";
|
|
575
|
+
say(o.dryRun
|
|
576
|
+
? `Would file ${count("file")}; ${count("already-filed")} already filed${overText}.`
|
|
577
|
+
: `Filed ${filed.length}; ${count("already-filed")} already filed${updated.length ? `, ${updated.length} of them updated` : ""}${overText}.`);
|
|
578
|
+
if (problems.length > 0) {
|
|
579
|
+
for (const p of problems)
|
|
580
|
+
complain(`scenescout export: ${p}`);
|
|
581
|
+
return { exitCode: EXIT_EXPORT.couldNotExport, plan, filed, updated };
|
|
582
|
+
}
|
|
583
|
+
return { exitCode: EXIT_EXPORT.done, plan, filed, updated };
|
|
584
|
+
}
|
|
585
|
+
catch (err) {
|
|
586
|
+
reportFrames();
|
|
587
|
+
complain(`scenescout export: ${err instanceof Error ? err.message : String(err)}`);
|
|
588
|
+
if (err instanceof ExportError && err.uncertain)
|
|
589
|
+
complain(o.to === "jira"
|
|
590
|
+
? "Jira may have made that issue before the error, and its search can take minutes to show a new one, so it was not sent again. Export again later: the issue is found by its marker if it exists, and filed if it does not."
|
|
591
|
+
: "The tracker may have made that issue before the error. The next export finds it by its marker and does not file it again.");
|
|
592
|
+
for (const p of problems)
|
|
593
|
+
complain(`scenescout export: ${p}`);
|
|
594
|
+
if (filed.length > 0)
|
|
595
|
+
complain(`Filed before it stopped: ${filed.map((x) => x.issue.ref).join(", ")}.`);
|
|
596
|
+
if (updated.length > 0)
|
|
597
|
+
complain(`Updated before it stopped: ${updated.map((x) => x.issue.ref).join(", ")}.`);
|
|
598
|
+
return { exitCode: EXIT_EXPORT.couldNotExport, plan, filed, updated };
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
const GITHUB_HEADERS = { accept: "application/vnd.github+json", "x-github-api-version": "2022-11-28" };
|
|
602
|
+
/**
|
|
603
|
+
* Create one issue. A create that failed in a way the tracker may have acted
|
|
604
|
+
* on (a timeout, a 5xx, an answer cut off) is not simply sent again: the
|
|
605
|
+
* issues are listed again first, and if one already carries this finding's
|
|
606
|
+
* marker, that is the issue. Only when none does is the create repeated, and
|
|
607
|
+
* only where the listing shows a new issue at once (GitHub); in Jira, whose
|
|
608
|
+
* search lags, the export stops instead.
|
|
609
|
+
*/
|
|
610
|
+
async function fileOne(tracker, f, ctx, extras, includeClosed, http) {
|
|
611
|
+
for (let attempt = 0;; attempt++) {
|
|
612
|
+
try {
|
|
613
|
+
return await tracker.create(f, ctx, extras);
|
|
614
|
+
}
|
|
615
|
+
catch (err) {
|
|
616
|
+
if (!(err instanceof ExportError) || !err.uncertain)
|
|
617
|
+
throw err;
|
|
618
|
+
const which = `the issue for finding ${f.id} (${oneLine(f.title, 80)})`;
|
|
619
|
+
if (!tracker.consistentListing || attempt >= http.retries)
|
|
620
|
+
throw new ExportError(`${err.message}, creating ${which}`, true);
|
|
621
|
+
await http.wait(backoffMs(attempt + 1, http.random));
|
|
622
|
+
let issue;
|
|
623
|
+
try {
|
|
624
|
+
issue = (await tracker.existing(includeClosed)).map.get(f.id);
|
|
625
|
+
}
|
|
626
|
+
catch (listing) {
|
|
627
|
+
throw new ExportError(`${err.message}, creating ${which}; listing the issues again to look for it failed too: ${listing instanceof Error ? listing.message : String(listing)}`, true);
|
|
628
|
+
}
|
|
629
|
+
if (issue)
|
|
630
|
+
return { issue, problems: [], confirmed: true };
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
}
|