seo-crawl-audit 0.1.2 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +335 -228
- package/bin/seo-audit.js +18 -3
- package/bundle/cli.js +154 -0
- package/package.json +10 -8
- package/examples/quotes-toscrape-report.html +0 -423
- package/src/audit.js +0 -125
- package/src/baseline.js +0 -65
- package/src/checkpoint.js +0 -90
- package/src/cli.js +0 -850
- package/src/compare.js +0 -205
- package/src/crawler.js +0 -306
- package/src/html-report.js +0 -474
- package/src/html.js +0 -105
- package/src/report.js +0 -43
- package/src/request-gate.js +0 -27
- package/src/robots.js +0 -80
- package/src/sitemap.js +0 -212
- package/src/target.js +0 -35
- package/src/urls.js +0 -45
package/src/cli.js
DELETED
|
@@ -1,850 +0,0 @@
|
|
|
1
|
-
import { parseArgs } from "node:util";
|
|
2
|
-
import { resolve } from "node:path";
|
|
3
|
-
import { createInterface } from "node:readline/promises";
|
|
4
|
-
import {
|
|
5
|
-
createBaseline,
|
|
6
|
-
readBaseline,
|
|
7
|
-
writeBaseline,
|
|
8
|
-
} from "./baseline.js";
|
|
9
|
-
import { auditBaseline } from "./audit.js";
|
|
10
|
-
import {
|
|
11
|
-
appendCheckpointPages,
|
|
12
|
-
checkpointPathForOutput,
|
|
13
|
-
initializeCheckpoint,
|
|
14
|
-
removeCheckpoint,
|
|
15
|
-
} from "./checkpoint.js";
|
|
16
|
-
import { compareBaselines } from "./compare.js";
|
|
17
|
-
import { crawlSite, fetchPages } from "./crawler.js";
|
|
18
|
-
import { writeHtmlReport } from "./html-report.js";
|
|
19
|
-
import { printIssues, summarizeIssues } from "./report.js";
|
|
20
|
-
import { createRequestGate } from "./request-gate.js";
|
|
21
|
-
import {
|
|
22
|
-
discoverSitemapUrl,
|
|
23
|
-
loadSitemapUrls,
|
|
24
|
-
} from "./sitemap.js";
|
|
25
|
-
import { mapUrlToBaseline, mapUrlToTarget } from "./target.js";
|
|
26
|
-
import { normalizeUrl } from "./urls.js";
|
|
27
|
-
|
|
28
|
-
const VERSION = "0.1.2";
|
|
29
|
-
const DEFAULT_BASELINE = ".seo-audit.json";
|
|
30
|
-
const DEFAULT_REPORT = "seo-audit-report.html";
|
|
31
|
-
|
|
32
|
-
const HELP = `seo-audit ${VERSION}
|
|
33
|
-
|
|
34
|
-
Local-first SEO crawler, audit, and regression checker.
|
|
35
|
-
|
|
36
|
-
Usage:
|
|
37
|
-
seo-audit <url> [options]
|
|
38
|
-
seo-audit scan <url> [options]
|
|
39
|
-
seo-audit check [url] [options]
|
|
40
|
-
seo-audit report [baseline] [options]
|
|
41
|
-
|
|
42
|
-
Commands:
|
|
43
|
-
<url> Shortcut for scan.
|
|
44
|
-
scan Crawl a site and save its SEO baseline.
|
|
45
|
-
check Crawl again and compare with a saved baseline.
|
|
46
|
-
report Generate HTML from an existing baseline without crawling.
|
|
47
|
-
|
|
48
|
-
Options:
|
|
49
|
-
--baseline <file> Baseline file for check (default: .seo-audit.json)
|
|
50
|
-
--output <file> Output file for scan (default: .seo-audit.json)
|
|
51
|
-
--report <file> HTML report path (default: seo-audit-report.html)
|
|
52
|
-
--no-report Do not generate the automatic HTML report
|
|
53
|
-
--no-cache Disable scan checkpoint caching and resume
|
|
54
|
-
--pages <number> Scan an exact number of pages
|
|
55
|
-
--all Scan every URL found in the sitemap
|
|
56
|
-
--max-pages <number> Deprecated alias for --pages
|
|
57
|
-
--concurrency <number> Concurrent requests (default: 5)
|
|
58
|
-
--delay <ms> Delay between request starts (default: 100)
|
|
59
|
-
--timeout <ms> Request timeout in milliseconds (default: 10000)
|
|
60
|
-
--sitemap <url> Seed the crawl from a sitemap or sitemap index
|
|
61
|
-
--no-sitemap Skip sitemap discovery and crawl internal links
|
|
62
|
-
--include-query Treat query-string URLs as separate pages
|
|
63
|
-
--ignore-robots Crawl URLs disallowed by robots.txt
|
|
64
|
-
--strict Fail check on warnings as well as errors
|
|
65
|
-
--json Print machine-readable command output
|
|
66
|
-
--help Show this help
|
|
67
|
-
--version Show the version
|
|
68
|
-
|
|
69
|
-
Exit codes:
|
|
70
|
-
0 No blocking regressions
|
|
71
|
-
1 SEO regressions detected
|
|
72
|
-
2 Invalid input or crawler failure
|
|
73
|
-
`;
|
|
74
|
-
|
|
75
|
-
function positiveInteger(value, name, fallback) {
|
|
76
|
-
if (value === undefined) {
|
|
77
|
-
return fallback;
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
const parsed = Number.parseInt(value, 10);
|
|
81
|
-
if (!Number.isInteger(parsed) || parsed <= 0) {
|
|
82
|
-
throw new Error(`${name} must be a positive integer`);
|
|
83
|
-
}
|
|
84
|
-
return parsed;
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
function nonNegativeInteger(value, name, fallback) {
|
|
88
|
-
if (value === undefined) {
|
|
89
|
-
return fallback;
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
const parsed = Number.parseInt(value, 10);
|
|
93
|
-
if (!Number.isInteger(parsed) || parsed < 0) {
|
|
94
|
-
throw new Error(`${name} must be a non-negative integer`);
|
|
95
|
-
}
|
|
96
|
-
return parsed;
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
function parseCliArgs(args) {
|
|
100
|
-
return parseArgs({
|
|
101
|
-
args,
|
|
102
|
-
allowPositionals: true,
|
|
103
|
-
strict: true,
|
|
104
|
-
options: {
|
|
105
|
-
baseline: { type: "string" },
|
|
106
|
-
output: { type: "string" },
|
|
107
|
-
report: { type: "string" },
|
|
108
|
-
"no-report": { type: "boolean" },
|
|
109
|
-
"no-cache": { type: "boolean" },
|
|
110
|
-
pages: { type: "string" },
|
|
111
|
-
all: { type: "boolean" },
|
|
112
|
-
"max-pages": { type: "string" },
|
|
113
|
-
concurrency: { type: "string" },
|
|
114
|
-
delay: { type: "string" },
|
|
115
|
-
timeout: { type: "string" },
|
|
116
|
-
sitemap: { type: "string" },
|
|
117
|
-
"no-sitemap": { type: "boolean" },
|
|
118
|
-
"include-query": { type: "boolean" },
|
|
119
|
-
"ignore-robots": { type: "boolean" },
|
|
120
|
-
strict: { type: "boolean" },
|
|
121
|
-
json: { type: "boolean" },
|
|
122
|
-
help: { type: "boolean" },
|
|
123
|
-
version: { type: "boolean" },
|
|
124
|
-
},
|
|
125
|
-
});
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
function shouldGenerateReport(values, force = false) {
|
|
129
|
-
if (force) {
|
|
130
|
-
return true;
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
if (values["no-report"]) {
|
|
134
|
-
return false;
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
return (
|
|
138
|
-
values.report !== undefined ||
|
|
139
|
-
(process.stdin.isTTY && process.stdout.isTTY && !values.json)
|
|
140
|
-
);
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
function getReportOutput(values) {
|
|
144
|
-
return resolve(values.report ?? DEFAULT_REPORT);
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
async function generateReport(values, payload, force = false) {
|
|
148
|
-
if (!shouldGenerateReport(values, force)) {
|
|
149
|
-
return null;
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
const output = getReportOutput(values);
|
|
153
|
-
await writeHtmlReport(output, payload);
|
|
154
|
-
return output;
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
function crawlerOptions(values, baseline) {
|
|
158
|
-
return {
|
|
159
|
-
maxPages: positiveInteger(
|
|
160
|
-
values.pages ?? values["max-pages"],
|
|
161
|
-
values.pages !== undefined ? "--pages" : "--max-pages",
|
|
162
|
-
baseline?.source.maxPages ?? 100,
|
|
163
|
-
),
|
|
164
|
-
concurrency: positiveInteger(values.concurrency, "--concurrency", 5),
|
|
165
|
-
requestDelay: nonNegativeInteger(
|
|
166
|
-
values.delay,
|
|
167
|
-
"--delay",
|
|
168
|
-
baseline?.source.requestDelay ?? 100,
|
|
169
|
-
),
|
|
170
|
-
timeout: positiveInteger(values.timeout, "--timeout", 10_000),
|
|
171
|
-
includeQuery:
|
|
172
|
-
values["include-query"] ?? baseline?.source.includeQuery ?? false,
|
|
173
|
-
respectRobots:
|
|
174
|
-
values["ignore-robots"] === undefined
|
|
175
|
-
? (baseline?.source.respectRobots ?? true)
|
|
176
|
-
: !values["ignore-robots"],
|
|
177
|
-
sitemap: values["no-sitemap"]
|
|
178
|
-
? null
|
|
179
|
-
: (values.sitemap ?? baseline?.source.sitemap ?? null),
|
|
180
|
-
};
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
function getHealthSummary(baseline) {
|
|
184
|
-
return baseline.pages.reduce(
|
|
185
|
-
(summary, page) => {
|
|
186
|
-
if (page.error || page.blockedByRobots || (page.status ?? 500) >= 400) {
|
|
187
|
-
summary.unavailable += 1;
|
|
188
|
-
}
|
|
189
|
-
if (!page.title) {
|
|
190
|
-
summary.missingTitle += 1;
|
|
191
|
-
}
|
|
192
|
-
if (!page.description) {
|
|
193
|
-
summary.missingDescription += 1;
|
|
194
|
-
}
|
|
195
|
-
if (!page.canonical) {
|
|
196
|
-
summary.missingCanonical += 1;
|
|
197
|
-
}
|
|
198
|
-
if ((page.h1Count ?? 0) === 0) {
|
|
199
|
-
summary.missingH1 += 1;
|
|
200
|
-
}
|
|
201
|
-
if (/(^|[\s,])noindex($|[\s,])/i.test(page.robots ?? "")) {
|
|
202
|
-
summary.noindex += 1;
|
|
203
|
-
}
|
|
204
|
-
return summary;
|
|
205
|
-
},
|
|
206
|
-
{
|
|
207
|
-
unavailable: 0,
|
|
208
|
-
missingTitle: 0,
|
|
209
|
-
missingDescription: 0,
|
|
210
|
-
missingCanonical: 0,
|
|
211
|
-
missingH1: 0,
|
|
212
|
-
noindex: 0,
|
|
213
|
-
},
|
|
214
|
-
);
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
export function parseScanMenuSelection(input, totalPages) {
|
|
218
|
-
const normalized = input.trim().toLowerCase();
|
|
219
|
-
|
|
220
|
-
if (normalized === "" || normalized === "1") {
|
|
221
|
-
return { mode: "fixed", target: Math.min(100, totalPages) };
|
|
222
|
-
}
|
|
223
|
-
if (normalized === "2" || normalized === "all") {
|
|
224
|
-
return { mode: "all", target: totalPages };
|
|
225
|
-
}
|
|
226
|
-
if (normalized === "3" || normalized === "step") {
|
|
227
|
-
return { mode: "step", target: totalPages };
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
const number = Number.parseInt(normalized, 10);
|
|
231
|
-
if (/^\d+$/.test(normalized) && number > 0) {
|
|
232
|
-
return { mode: "fixed", target: Math.min(number, totalPages) };
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
return null;
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
function printHealth(health) {
|
|
239
|
-
console.log(
|
|
240
|
-
`Current health: ${health.unavailable} unavailable, ${health.missingTitle} missing title, ${health.missingDescription} missing description, ${health.missingCanonical} missing canonical, ${health.missingH1} missing H1, ${health.noindex} noindex.`,
|
|
241
|
-
);
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
async function chooseScanPlan(totalPages) {
|
|
245
|
-
while (true) {
|
|
246
|
-
console.log(`\nSitemap contains ${totalPages.toLocaleString("en-US")} page(s).`);
|
|
247
|
-
console.log("How many pages should be scanned?");
|
|
248
|
-
console.log(" 1) First 100");
|
|
249
|
-
console.log(` 2) All ${totalPages.toLocaleString("en-US")}`);
|
|
250
|
-
console.log(" 3) 100 at a time with confirmation");
|
|
251
|
-
console.log(" Or enter an exact number, for example 500.");
|
|
252
|
-
|
|
253
|
-
const selection = parseScanMenuSelection(
|
|
254
|
-
await ask("Choose [1]: "),
|
|
255
|
-
totalPages,
|
|
256
|
-
);
|
|
257
|
-
if (!selection) {
|
|
258
|
-
console.log("Enter 1, 2, 3, all, step, or a positive number.");
|
|
259
|
-
continue;
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
if (selection.mode === "all") {
|
|
263
|
-
const confirmation = await ask(
|
|
264
|
-
`This will request ${totalPages.toLocaleString("en-US")} pages. Continue? [y/N] `,
|
|
265
|
-
);
|
|
266
|
-
if (!["y", "yes"].includes(confirmation.toLowerCase())) {
|
|
267
|
-
continue;
|
|
268
|
-
}
|
|
269
|
-
}
|
|
270
|
-
|
|
271
|
-
return selection;
|
|
272
|
-
}
|
|
273
|
-
}
|
|
274
|
-
|
|
275
|
-
function printProgress(checked, target, { persistent = false } = {}) {
|
|
276
|
-
const ratio = target === 0 ? 1 : Math.min(checked / target, 1);
|
|
277
|
-
const percent = (ratio * 100).toFixed(ratio < 0.1 ? 1 : 0);
|
|
278
|
-
const filled = Math.round(ratio * 20);
|
|
279
|
-
const bar = `${"█".repeat(filled)}${"░".repeat(20 - filled)}`;
|
|
280
|
-
const message = `[${bar}] ${percent}% · ${checked.toLocaleString("en-US")} / ${target.toLocaleString("en-US")} pages`;
|
|
281
|
-
|
|
282
|
-
if (process.stdout.isTTY && !persistent) {
|
|
283
|
-
process.stdout.write(`\r${message}`);
|
|
284
|
-
if (checked >= target) {
|
|
285
|
-
process.stdout.write("\n");
|
|
286
|
-
}
|
|
287
|
-
} else if (
|
|
288
|
-
process.stdout.isTTY ||
|
|
289
|
-
checked >= target ||
|
|
290
|
-
checked % 1_000 === 0
|
|
291
|
-
) {
|
|
292
|
-
console.log(message);
|
|
293
|
-
}
|
|
294
|
-
}
|
|
295
|
-
|
|
296
|
-
async function ask(question) {
|
|
297
|
-
const readline = createInterface({
|
|
298
|
-
input: process.stdin,
|
|
299
|
-
output: process.stdout,
|
|
300
|
-
});
|
|
301
|
-
|
|
302
|
-
try {
|
|
303
|
-
return (await readline.question(question)).trim();
|
|
304
|
-
} finally {
|
|
305
|
-
readline.close();
|
|
306
|
-
}
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
async function resolveSitemap(startUrl, values, options) {
|
|
310
|
-
if (values.sitemap && values["no-sitemap"]) {
|
|
311
|
-
throw new Error("--sitemap and --no-sitemap cannot be used together");
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
if (values["no-sitemap"]) {
|
|
315
|
-
return null;
|
|
316
|
-
}
|
|
317
|
-
|
|
318
|
-
if (values.sitemap) {
|
|
319
|
-
return values.sitemap;
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
const discovered = await discoverSitemapUrl(startUrl, {
|
|
323
|
-
timeout: options.timeout,
|
|
324
|
-
userAgent: "seo-crawl-audit/0.1.2",
|
|
325
|
-
requestGate: options.requestGate,
|
|
326
|
-
});
|
|
327
|
-
if (discovered) {
|
|
328
|
-
if (!values.json) {
|
|
329
|
-
console.log(`Found sitemap: ${discovered}`);
|
|
330
|
-
}
|
|
331
|
-
return discovered;
|
|
332
|
-
}
|
|
333
|
-
|
|
334
|
-
if (process.stdin.isTTY && process.stdout.isTTY && !values.json) {
|
|
335
|
-
const entered = await ask(
|
|
336
|
-
"Sitemap was not found. Enter its full URL, or press Enter to crawl internal links: ",
|
|
337
|
-
);
|
|
338
|
-
return entered || null;
|
|
339
|
-
}
|
|
340
|
-
|
|
341
|
-
return null;
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
async function scanCommand(url, values) {
|
|
345
|
-
if (!url) {
|
|
346
|
-
throw new Error("scan requires a URL");
|
|
347
|
-
}
|
|
348
|
-
|
|
349
|
-
const output = resolve(values.output ?? DEFAULT_BASELINE);
|
|
350
|
-
const options = crawlerOptions(values);
|
|
351
|
-
options.requestGate = createRequestGate(options.requestDelay);
|
|
352
|
-
const normalizedStartUrl = normalizeUrl(url, undefined, {
|
|
353
|
-
includeQuery: options.includeQuery,
|
|
354
|
-
});
|
|
355
|
-
if (!normalizedStartUrl) {
|
|
356
|
-
throw new Error(`invalid start URL: ${url}`);
|
|
357
|
-
}
|
|
358
|
-
options.sitemap = await resolveSitemap(
|
|
359
|
-
normalizedStartUrl,
|
|
360
|
-
values,
|
|
361
|
-
options,
|
|
362
|
-
);
|
|
363
|
-
const isInteractive =
|
|
364
|
-
process.stdin.isTTY && process.stdout.isTTY && !values.json;
|
|
365
|
-
let scanPlan = {
|
|
366
|
-
mode: "fixed",
|
|
367
|
-
target: options.maxPages,
|
|
368
|
-
};
|
|
369
|
-
|
|
370
|
-
if (options.sitemap) {
|
|
371
|
-
const startUrl = normalizedStartUrl;
|
|
372
|
-
options.sitemapData = await loadSitemapUrls(options.sitemap, {
|
|
373
|
-
...options,
|
|
374
|
-
maxUrls: 50_000,
|
|
375
|
-
siteOrigin: new URL(startUrl).origin,
|
|
376
|
-
});
|
|
377
|
-
const totalCandidates = new Set([
|
|
378
|
-
startUrl,
|
|
379
|
-
...options.sitemapData.urls,
|
|
380
|
-
]).size;
|
|
381
|
-
|
|
382
|
-
if (values.all) {
|
|
383
|
-
scanPlan = { mode: "all", target: totalCandidates };
|
|
384
|
-
} else if (
|
|
385
|
-
values.pages !== undefined ||
|
|
386
|
-
values["max-pages"] !== undefined
|
|
387
|
-
) {
|
|
388
|
-
scanPlan = {
|
|
389
|
-
mode: "fixed",
|
|
390
|
-
target: Math.min(options.maxPages, totalCandidates),
|
|
391
|
-
};
|
|
392
|
-
} else if (isInteractive) {
|
|
393
|
-
scanPlan = await chooseScanPlan(totalCandidates);
|
|
394
|
-
} else {
|
|
395
|
-
scanPlan = {
|
|
396
|
-
mode: "fixed",
|
|
397
|
-
target: Math.min(100, totalCandidates),
|
|
398
|
-
};
|
|
399
|
-
}
|
|
400
|
-
} else if (values.all) {
|
|
401
|
-
throw new Error("--all requires a sitemap");
|
|
402
|
-
}
|
|
403
|
-
|
|
404
|
-
const batchSize = Math.min(100, scanPlan.target);
|
|
405
|
-
options.maxPages = batchSize;
|
|
406
|
-
const checkpointEnabled = !values["no-cache"];
|
|
407
|
-
const checkpointPath = checkpointPathForOutput(output);
|
|
408
|
-
const checkpointSource = {
|
|
409
|
-
startUrl: normalizedStartUrl,
|
|
410
|
-
sitemap: options.sitemap,
|
|
411
|
-
includeQuery: options.includeQuery,
|
|
412
|
-
respectRobots: options.respectRobots,
|
|
413
|
-
};
|
|
414
|
-
const checkpoint = checkpointEnabled
|
|
415
|
-
? await initializeCheckpoint(checkpointPath, checkpointSource)
|
|
416
|
-
: { pages: [], resumed: false };
|
|
417
|
-
const plannedUrls = options.sitemapData
|
|
418
|
-
? new Set(
|
|
419
|
-
[
|
|
420
|
-
normalizedStartUrl,
|
|
421
|
-
...options.sitemapData.urls,
|
|
422
|
-
].slice(0, scanPlan.target),
|
|
423
|
-
)
|
|
424
|
-
: null;
|
|
425
|
-
const cachedPages = checkpoint.pages
|
|
426
|
-
.filter((page) => !plannedUrls || plannedUrls.has(page.url))
|
|
427
|
-
.slice(0, scanPlan.target);
|
|
428
|
-
const savedPages = new Map(
|
|
429
|
-
cachedPages.map((page) => [page.url, page]),
|
|
430
|
-
);
|
|
431
|
-
const reportEnabled = shouldGenerateReport(values);
|
|
432
|
-
const partialReportOutput = reportEnabled
|
|
433
|
-
? getReportOutput(values)
|
|
434
|
-
: null;
|
|
435
|
-
let lastReportPageCount = -1;
|
|
436
|
-
let lastReportAt = 0;
|
|
437
|
-
|
|
438
|
-
if (savedPages.size > 0 && !values.json) {
|
|
439
|
-
console.log(
|
|
440
|
-
`Resuming from checkpoint: ${savedPages.size.toLocaleString("en-US")} page(s) already saved.`,
|
|
441
|
-
);
|
|
442
|
-
}
|
|
443
|
-
|
|
444
|
-
const writePartialReport = async (force = false) => {
|
|
445
|
-
if (!partialReportOutput) {
|
|
446
|
-
return;
|
|
447
|
-
}
|
|
448
|
-
|
|
449
|
-
const now = Date.now();
|
|
450
|
-
const hasFirstResults =
|
|
451
|
-
lastReportPageCount === 0 && savedPages.size > 0;
|
|
452
|
-
if (
|
|
453
|
-
!force &&
|
|
454
|
-
!hasFirstResults &&
|
|
455
|
-
lastReportPageCount >= 0 &&
|
|
456
|
-
savedPages.size - lastReportPageCount < 100 &&
|
|
457
|
-
now - lastReportAt < 5_000
|
|
458
|
-
) {
|
|
459
|
-
return;
|
|
460
|
-
}
|
|
461
|
-
|
|
462
|
-
const pages = [...savedPages.values()].sort((left, right) =>
|
|
463
|
-
left.url.localeCompare(right.url),
|
|
464
|
-
);
|
|
465
|
-
await writeHtmlReport(partialReportOutput, {
|
|
466
|
-
mode: "scan",
|
|
467
|
-
startUrl: normalizedStartUrl,
|
|
468
|
-
generatedAt: new Date().toISOString(),
|
|
469
|
-
pages,
|
|
470
|
-
issues: auditBaseline({ pages }),
|
|
471
|
-
partial: true,
|
|
472
|
-
targetPages: scanPlan.target,
|
|
473
|
-
});
|
|
474
|
-
lastReportPageCount = savedPages.size;
|
|
475
|
-
lastReportAt = now;
|
|
476
|
-
};
|
|
477
|
-
|
|
478
|
-
const saveFreshPages = async (pages, forceReport = false) => {
|
|
479
|
-
const fresh = pages.filter((page) => !savedPages.has(page.url));
|
|
480
|
-
for (const page of fresh) {
|
|
481
|
-
savedPages.set(page.url, page);
|
|
482
|
-
}
|
|
483
|
-
if (checkpointEnabled) {
|
|
484
|
-
await appendCheckpointPages(checkpointPath, fresh);
|
|
485
|
-
}
|
|
486
|
-
await writePartialReport(forceReport);
|
|
487
|
-
};
|
|
488
|
-
|
|
489
|
-
await writePartialReport(true);
|
|
490
|
-
|
|
491
|
-
const scan = await crawlSite(url, {
|
|
492
|
-
...options,
|
|
493
|
-
cachedPages,
|
|
494
|
-
onBatch: saveFreshPages,
|
|
495
|
-
onProgress:
|
|
496
|
-
!values.json && scanPlan.mode !== "step"
|
|
497
|
-
? (checkedPages) =>
|
|
498
|
-
printProgress(
|
|
499
|
-
Math.min(
|
|
500
|
-
Math.max(savedPages.size, checkedPages),
|
|
501
|
-
scanPlan.target,
|
|
502
|
-
),
|
|
503
|
-
scanPlan.target,
|
|
504
|
-
)
|
|
505
|
-
: undefined,
|
|
506
|
-
});
|
|
507
|
-
const checked = new Map(savedPages);
|
|
508
|
-
for (const page of scan.pages) {
|
|
509
|
-
checked.set(page.url, page);
|
|
510
|
-
}
|
|
511
|
-
if (
|
|
512
|
-
!values.json &&
|
|
513
|
-
process.stdout.isTTY &&
|
|
514
|
-
scanPlan.mode !== "step" &&
|
|
515
|
-
!scan.sitemap &&
|
|
516
|
-
checked.size < scanPlan.target
|
|
517
|
-
) {
|
|
518
|
-
process.stdout.write("\n");
|
|
519
|
-
}
|
|
520
|
-
const allCandidates = [
|
|
521
|
-
...new Set([scan.startUrl, ...(scan.sitemap?.urls ?? [])]),
|
|
522
|
-
];
|
|
523
|
-
let pendingUrls = allCandidates.filter((pageUrl) => !checked.has(pageUrl));
|
|
524
|
-
|
|
525
|
-
if (
|
|
526
|
-
scan.sitemap &&
|
|
527
|
-
scanPlan.mode === "step" &&
|
|
528
|
-
scanPlan.target > checked.size &&
|
|
529
|
-
!values.json
|
|
530
|
-
) {
|
|
531
|
-
printProgress(checked.size, scanPlan.target, {
|
|
532
|
-
persistent: scanPlan.mode === "step",
|
|
533
|
-
});
|
|
534
|
-
}
|
|
535
|
-
|
|
536
|
-
while (
|
|
537
|
-
scan.sitemap &&
|
|
538
|
-
pendingUrls.length > 0 &&
|
|
539
|
-
checked.size < scanPlan.target
|
|
540
|
-
) {
|
|
541
|
-
if (scanPlan.mode === "step") {
|
|
542
|
-
printHealth(
|
|
543
|
-
getHealthSummary({
|
|
544
|
-
pages: [...checked.values()],
|
|
545
|
-
}),
|
|
546
|
-
);
|
|
547
|
-
const nextSize = Math.min(
|
|
548
|
-
100,
|
|
549
|
-
scanPlan.target - checked.size,
|
|
550
|
-
pendingUrls.length,
|
|
551
|
-
);
|
|
552
|
-
const answer = await ask(
|
|
553
|
-
`Check the next ${nextSize} page(s)? [y/N] `,
|
|
554
|
-
);
|
|
555
|
-
if (!["y", "yes"].includes(answer.toLowerCase())) {
|
|
556
|
-
break;
|
|
557
|
-
}
|
|
558
|
-
}
|
|
559
|
-
|
|
560
|
-
const batch = pendingUrls.slice(
|
|
561
|
-
0,
|
|
562
|
-
Math.min(100, scanPlan.target - checked.size),
|
|
563
|
-
);
|
|
564
|
-
const additional = await fetchPages(batch, {
|
|
565
|
-
...options,
|
|
566
|
-
robots: scan.robots,
|
|
567
|
-
onBatch: saveFreshPages,
|
|
568
|
-
});
|
|
569
|
-
for (const page of additional.pages) {
|
|
570
|
-
checked.set(page.url, page);
|
|
571
|
-
}
|
|
572
|
-
await writePartialReport(true);
|
|
573
|
-
pendingUrls = allCandidates.filter(
|
|
574
|
-
(pageUrl) => !checked.has(pageUrl),
|
|
575
|
-
);
|
|
576
|
-
|
|
577
|
-
if (!values.json) {
|
|
578
|
-
printProgress(checked.size, scanPlan.target, {
|
|
579
|
-
persistent: scanPlan.mode === "step",
|
|
580
|
-
});
|
|
581
|
-
}
|
|
582
|
-
}
|
|
583
|
-
|
|
584
|
-
scan.pages = [...checked.values()].sort((left, right) =>
|
|
585
|
-
left.url.localeCompare(right.url),
|
|
586
|
-
);
|
|
587
|
-
scan.options.maxPages = scanPlan.target;
|
|
588
|
-
scan.truncated =
|
|
589
|
-
checked.size < allCandidates.length || (scan.sitemap?.truncated ?? false);
|
|
590
|
-
const baseline = createBaseline(scan);
|
|
591
|
-
const health = getHealthSummary(baseline);
|
|
592
|
-
await writeBaseline(output, baseline);
|
|
593
|
-
const targetCompleted =
|
|
594
|
-
checked.size >= scanPlan.target || pendingUrls.length === 0;
|
|
595
|
-
let reportOutput;
|
|
596
|
-
if (targetCompleted) {
|
|
597
|
-
if (checkpointEnabled) {
|
|
598
|
-
await removeCheckpoint(checkpointPath);
|
|
599
|
-
}
|
|
600
|
-
reportOutput = await generateReport(values, {
|
|
601
|
-
mode: "scan",
|
|
602
|
-
startUrl: baseline.source.startUrl,
|
|
603
|
-
generatedAt: new Date().toISOString(),
|
|
604
|
-
pages: baseline.pages,
|
|
605
|
-
issues: auditBaseline(baseline),
|
|
606
|
-
});
|
|
607
|
-
} else {
|
|
608
|
-
await writePartialReport(true);
|
|
609
|
-
reportOutput = partialReportOutput;
|
|
610
|
-
}
|
|
611
|
-
|
|
612
|
-
if (values.json) {
|
|
613
|
-
console.log(
|
|
614
|
-
JSON.stringify(
|
|
615
|
-
{
|
|
616
|
-
command: "scan",
|
|
617
|
-
pages: baseline.pages.length,
|
|
618
|
-
truncated: baseline.truncated,
|
|
619
|
-
health,
|
|
620
|
-
output,
|
|
621
|
-
report: reportOutput,
|
|
622
|
-
checkpoint: targetCompleted || !checkpointEnabled
|
|
623
|
-
? null
|
|
624
|
-
: checkpointPath,
|
|
625
|
-
},
|
|
626
|
-
null,
|
|
627
|
-
2,
|
|
628
|
-
),
|
|
629
|
-
);
|
|
630
|
-
} else {
|
|
631
|
-
console.log(
|
|
632
|
-
`Crawled ${baseline.pages.length} page(s)${baseline.truncated ? " (limit reached)" : ""}.`,
|
|
633
|
-
);
|
|
634
|
-
if (baseline.sitemap) {
|
|
635
|
-
console.log(
|
|
636
|
-
`Loaded ${baseline.sitemap.sitemapCount} sitemap file(s) from ${baseline.sitemap.url}.`,
|
|
637
|
-
);
|
|
638
|
-
}
|
|
639
|
-
printHealth(health);
|
|
640
|
-
console.log(`Baseline saved to ${output}`);
|
|
641
|
-
if (reportOutput) {
|
|
642
|
-
console.log(`HTML report saved to ${reportOutput}`);
|
|
643
|
-
}
|
|
644
|
-
if (!targetCompleted && checkpointEnabled) {
|
|
645
|
-
console.log(`Resume checkpoint saved to ${checkpointPath}`);
|
|
646
|
-
}
|
|
647
|
-
}
|
|
648
|
-
|
|
649
|
-
return 0;
|
|
650
|
-
}
|
|
651
|
-
|
|
652
|
-
async function checkCommand(url, values) {
|
|
653
|
-
const baselinePath = resolve(values.baseline ?? DEFAULT_BASELINE);
|
|
654
|
-
const baseline = await readBaseline(baselinePath);
|
|
655
|
-
const startUrl = url ?? baseline.source.startUrl;
|
|
656
|
-
const options = crawlerOptions(values, baseline);
|
|
657
|
-
options.requestGate = createRequestGate(options.requestDelay);
|
|
658
|
-
if (options.sitemap) {
|
|
659
|
-
options.sitemap = mapUrlToTarget(
|
|
660
|
-
options.sitemap,
|
|
661
|
-
baseline.source.startUrl,
|
|
662
|
-
startUrl,
|
|
663
|
-
);
|
|
664
|
-
}
|
|
665
|
-
const totalPages = baseline.pages.length;
|
|
666
|
-
const scan = await crawlSite(startUrl, {
|
|
667
|
-
...options,
|
|
668
|
-
onProgress: values.json
|
|
669
|
-
? undefined
|
|
670
|
-
: (checkedPages) => printProgress(
|
|
671
|
-
Math.min(checkedPages, totalPages),
|
|
672
|
-
totalPages,
|
|
673
|
-
),
|
|
674
|
-
});
|
|
675
|
-
const checked = new Map(scan.pages.map((page) => [page.url, page]));
|
|
676
|
-
const targets = baseline.pages.map((page) => ({
|
|
677
|
-
baselineUrl: page.url,
|
|
678
|
-
targetUrl: mapUrlToTarget(
|
|
679
|
-
page.url,
|
|
680
|
-
baseline.source.startUrl,
|
|
681
|
-
scan.startUrl,
|
|
682
|
-
),
|
|
683
|
-
}));
|
|
684
|
-
const missingUrls = targets
|
|
685
|
-
.map((target) => target.targetUrl)
|
|
686
|
-
.filter((targetUrl) => !checked.has(targetUrl));
|
|
687
|
-
|
|
688
|
-
if (missingUrls.length > 0) {
|
|
689
|
-
const additional = await fetchPages(missingUrls, {
|
|
690
|
-
...options,
|
|
691
|
-
robots: scan.robots,
|
|
692
|
-
onProgress: values.json
|
|
693
|
-
? undefined
|
|
694
|
-
: (checkedPages) =>
|
|
695
|
-
printProgress(
|
|
696
|
-
Math.min(scan.pages.length + checkedPages, totalPages),
|
|
697
|
-
totalPages,
|
|
698
|
-
),
|
|
699
|
-
});
|
|
700
|
-
for (const page of additional.pages) {
|
|
701
|
-
checked.set(page.url, page);
|
|
702
|
-
}
|
|
703
|
-
}
|
|
704
|
-
|
|
705
|
-
const comparablePages = targets
|
|
706
|
-
.map(({ baselineUrl, targetUrl }) => {
|
|
707
|
-
const page = checked.get(targetUrl);
|
|
708
|
-
if (!page) {
|
|
709
|
-
return null;
|
|
710
|
-
}
|
|
711
|
-
|
|
712
|
-
return {
|
|
713
|
-
...page,
|
|
714
|
-
url: baselineUrl,
|
|
715
|
-
finalUrl: mapUrlToBaseline(
|
|
716
|
-
page.finalUrl,
|
|
717
|
-
baseline.source.startUrl,
|
|
718
|
-
scan.startUrl,
|
|
719
|
-
),
|
|
720
|
-
};
|
|
721
|
-
})
|
|
722
|
-
.filter(Boolean);
|
|
723
|
-
const current = createBaseline({
|
|
724
|
-
...scan,
|
|
725
|
-
pages: comparablePages.sort((left, right) =>
|
|
726
|
-
left.url.localeCompare(right.url),
|
|
727
|
-
),
|
|
728
|
-
});
|
|
729
|
-
const issues = compareBaselines(baseline, current);
|
|
730
|
-
const summary = summarizeIssues(issues);
|
|
731
|
-
const reportOutput = await generateReport(values, {
|
|
732
|
-
mode: "check",
|
|
733
|
-
startUrl,
|
|
734
|
-
generatedAt: new Date().toISOString(),
|
|
735
|
-
pages: current.pages,
|
|
736
|
-
issues,
|
|
737
|
-
});
|
|
738
|
-
|
|
739
|
-
if (values.json) {
|
|
740
|
-
console.log(
|
|
741
|
-
JSON.stringify(
|
|
742
|
-
{
|
|
743
|
-
command: "check",
|
|
744
|
-
baseline: baselinePath,
|
|
745
|
-
pages: current.pages.length,
|
|
746
|
-
summary,
|
|
747
|
-
issues,
|
|
748
|
-
report: reportOutput,
|
|
749
|
-
},
|
|
750
|
-
null,
|
|
751
|
-
2,
|
|
752
|
-
),
|
|
753
|
-
);
|
|
754
|
-
} else {
|
|
755
|
-
console.log(`Checked ${current.pages.length} page(s).\n`);
|
|
756
|
-
printIssues(issues);
|
|
757
|
-
if (reportOutput) {
|
|
758
|
-
console.log(`HTML report saved to ${reportOutput}`);
|
|
759
|
-
}
|
|
760
|
-
}
|
|
761
|
-
|
|
762
|
-
return summary.error > 0 || (values.strict && summary.warning > 0) ? 1 : 0;
|
|
763
|
-
}
|
|
764
|
-
|
|
765
|
-
async function reportCommand(inputPath, values) {
|
|
766
|
-
const baselinePath = resolve(inputPath ?? values.baseline ?? DEFAULT_BASELINE);
|
|
767
|
-
const baseline = await readBaseline(baselinePath);
|
|
768
|
-
const reportOutput = await generateReport(
|
|
769
|
-
values,
|
|
770
|
-
{
|
|
771
|
-
mode: "scan",
|
|
772
|
-
startUrl: baseline.source.startUrl,
|
|
773
|
-
generatedAt: new Date().toISOString(),
|
|
774
|
-
pages: baseline.pages,
|
|
775
|
-
issues: auditBaseline(baseline),
|
|
776
|
-
},
|
|
777
|
-
true,
|
|
778
|
-
);
|
|
779
|
-
|
|
780
|
-
if (values.json) {
|
|
781
|
-
console.log(
|
|
782
|
-
JSON.stringify(
|
|
783
|
-
{
|
|
784
|
-
command: "report",
|
|
785
|
-
baseline: baselinePath,
|
|
786
|
-
pages: baseline.pages.length,
|
|
787
|
-
report: reportOutput,
|
|
788
|
-
},
|
|
789
|
-
null,
|
|
790
|
-
2,
|
|
791
|
-
),
|
|
792
|
-
);
|
|
793
|
-
} else {
|
|
794
|
-
console.log(`HTML report saved to ${reportOutput}`);
|
|
795
|
-
}
|
|
796
|
-
|
|
797
|
-
return 0;
|
|
798
|
-
}
|
|
799
|
-
|
|
800
|
-
export async function main(args = process.argv.slice(2)) {
|
|
801
|
-
let parsed;
|
|
802
|
-
|
|
803
|
-
try {
|
|
804
|
-
parsed = parseCliArgs(args);
|
|
805
|
-
} catch (error) {
|
|
806
|
-
console.error(`${error.message}\n\n${HELP}`);
|
|
807
|
-
return 2;
|
|
808
|
-
}
|
|
809
|
-
|
|
810
|
-
const { values, positionals } = parsed;
|
|
811
|
-
|
|
812
|
-
if (values.version) {
|
|
813
|
-
console.log(VERSION);
|
|
814
|
-
return 0;
|
|
815
|
-
}
|
|
816
|
-
|
|
817
|
-
if (values.help || positionals.length === 0) {
|
|
818
|
-
console.log(HELP);
|
|
819
|
-
return 0;
|
|
820
|
-
}
|
|
821
|
-
|
|
822
|
-
let [command, url, ...extra] = positionals;
|
|
823
|
-
if (/^https?:\/\//i.test(command)) {
|
|
824
|
-
extra = positionals.slice(1);
|
|
825
|
-
url = command;
|
|
826
|
-
command = "scan";
|
|
827
|
-
}
|
|
828
|
-
if (extra.length > 0) {
|
|
829
|
-
console.error(`Unexpected argument: ${extra[0]}`);
|
|
830
|
-
return 2;
|
|
831
|
-
}
|
|
832
|
-
|
|
833
|
-
try {
|
|
834
|
-
if (command === "scan") {
|
|
835
|
-
return await scanCommand(url, values);
|
|
836
|
-
}
|
|
837
|
-
if (command === "check") {
|
|
838
|
-
return await checkCommand(url, values);
|
|
839
|
-
}
|
|
840
|
-
if (command === "report") {
|
|
841
|
-
return await reportCommand(url, values);
|
|
842
|
-
}
|
|
843
|
-
|
|
844
|
-
console.error(`Unknown command: ${command}\n\n${HELP}`);
|
|
845
|
-
return 2;
|
|
846
|
-
} catch (error) {
|
|
847
|
-
console.error(`seo-audit: ${error.message}`);
|
|
848
|
-
return 2;
|
|
849
|
-
}
|
|
850
|
-
}
|