seo 0.2.13 → 0.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cli.js +2 -2
- package/dist/{dist-BB0yH7qK.js → dist-Bc8MJ1gT.js} +1 -1
- package/dist/{dist-Dwdogl4j.js → dist-sYeCtY3Q.js} +87 -87
- package/dist/dist-sYeCtY3Q.js.map +1 -0
- package/dist/index.d.ts +70 -2
- package/dist/index.js +88 -38
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +88 -38
- package/dist/mcp.js.map +1 -1
- package/dist/{package-B_swz2Xd.js → package-Cp8DwqCy.js} +2 -2
- package/dist/{package-B_swz2Xd.js.map → package-Cp8DwqCy.js.map} +1 -1
- package/package.json +1 -1
- package/dist/dist-Dwdogl4j.js.map +0 -1
package/dist/index.d.ts
CHANGED
|
@@ -980,7 +980,7 @@ declare const RULE_DEFINITIONS: readonly [{
|
|
|
980
980
|
readonly impactIfIgnored: "An avoidable redirect adds a request and can turn into a chain later. An intentional direct redirect needs no change.";
|
|
981
981
|
readonly howToVerify: "Re-run the crawl and confirm the requested URL is the same as the final URL, or that the redirect is intentional.";
|
|
982
982
|
readonly agentHints: {
|
|
983
|
-
readonly evidenceFields: ["page.url", "page.finalUrl", "page.status"];
|
|
983
|
+
readonly evidenceFields: ["page.url", "page.finalUrl", "page.status", "page.metaRefresh"];
|
|
984
984
|
readonly suggestedCommands: ["seo redirect-trace --url <url> --json"];
|
|
985
985
|
};
|
|
986
986
|
}, {
|
|
@@ -2012,6 +2012,10 @@ type CrawlerExtractionEvidence = {
|
|
|
2012
2012
|
}>;
|
|
2013
2013
|
socialProfileLinks: string[];
|
|
2014
2014
|
contentSketch: ContentSketch;
|
|
2015
|
+
metaRefresh?: {
|
|
2016
|
+
targetUrl: string;
|
|
2017
|
+
delaySeconds: number;
|
|
2018
|
+
};
|
|
2015
2019
|
softAuthenticationGate?: {
|
|
2016
2020
|
kind: 'login-form';
|
|
2017
2021
|
indicators: string[];
|
|
@@ -2495,7 +2499,7 @@ type CrawlPageSnapshot = {
|
|
|
2495
2499
|
h3Count?: number;
|
|
2496
2500
|
indexable: boolean;
|
|
2497
2501
|
indexability?: string;
|
|
2498
|
-
declaredIndexability?: 'indexable-candidate' | 'noindex' | 'robots-blocked' | 'canonical-conflict' | 'canonical-hint-other' | 'not-html' | 'unknown';
|
|
2502
|
+
declaredIndexability?: 'indexable-candidate' | 'noindex' | 'robots-blocked' | 'canonical-conflict' | 'canonical-hint-other' | 'meta-refresh' | 'not-html' | 'unknown';
|
|
2499
2503
|
extractionStatus?: 'complete' | 'failed' | 'not-applicable' | 'unknown-media-type';
|
|
2500
2504
|
extractionError?: string;
|
|
2501
2505
|
wordCount: number;
|
|
@@ -2511,6 +2515,10 @@ type CrawlPageSnapshot = {
|
|
|
2511
2515
|
indicators: string[];
|
|
2512
2516
|
formActionPath?: string;
|
|
2513
2517
|
};
|
|
2518
|
+
metaRefresh?: {
|
|
2519
|
+
targetUrl: string;
|
|
2520
|
+
delaySeconds: number;
|
|
2521
|
+
};
|
|
2514
2522
|
tabbedContent?: {
|
|
2515
2523
|
groups: number;
|
|
2516
2524
|
panels: number;
|
|
@@ -4697,6 +4705,7 @@ declare const crawlPageSnapshotSchema: z.ZodObject<{
|
|
|
4697
4705
|
"robots-blocked": "robots-blocked";
|
|
4698
4706
|
"canonical-conflict": "canonical-conflict";
|
|
4699
4707
|
"canonical-hint-other": "canonical-hint-other";
|
|
4708
|
+
"meta-refresh": "meta-refresh";
|
|
4700
4709
|
"not-html": "not-html";
|
|
4701
4710
|
}>>;
|
|
4702
4711
|
extractionStatus: z.ZodOptional<z.ZodEnum<{
|
|
@@ -4748,6 +4757,10 @@ declare const crawlPageSnapshotSchema: z.ZodObject<{
|
|
|
4748
4757
|
indicators: z.ZodArray<z.ZodString>;
|
|
4749
4758
|
formActionPath: z.ZodOptional<z.ZodString>;
|
|
4750
4759
|
}, z.core.$strip>>;
|
|
4760
|
+
metaRefresh: z.ZodOptional<z.ZodObject<{
|
|
4761
|
+
targetUrl: z.ZodString;
|
|
4762
|
+
delaySeconds: z.ZodNumber;
|
|
4763
|
+
}, z.core.$strip>>;
|
|
4751
4764
|
tabbedContent: z.ZodOptional<z.ZodObject<{
|
|
4752
4765
|
groups: z.ZodNumber;
|
|
4753
4766
|
panels: z.ZodNumber;
|
|
@@ -5856,6 +5869,7 @@ declare const crawlReportSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
5856
5869
|
"robots-blocked": "robots-blocked";
|
|
5857
5870
|
"canonical-conflict": "canonical-conflict";
|
|
5858
5871
|
"canonical-hint-other": "canonical-hint-other";
|
|
5872
|
+
"meta-refresh": "meta-refresh";
|
|
5859
5873
|
"not-html": "not-html";
|
|
5860
5874
|
}>>;
|
|
5861
5875
|
extractionStatus: z.ZodOptional<z.ZodEnum<{
|
|
@@ -5907,6 +5921,10 @@ declare const crawlReportSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
5907
5921
|
indicators: z.ZodArray<z.ZodString>;
|
|
5908
5922
|
formActionPath: z.ZodOptional<z.ZodString>;
|
|
5909
5923
|
}, z.core.$strip>>;
|
|
5924
|
+
metaRefresh: z.ZodOptional<z.ZodObject<{
|
|
5925
|
+
targetUrl: z.ZodString;
|
|
5926
|
+
delaySeconds: z.ZodNumber;
|
|
5927
|
+
}, z.core.$strip>>;
|
|
5910
5928
|
tabbedContent: z.ZodOptional<z.ZodObject<{
|
|
5911
5929
|
groups: z.ZodNumber;
|
|
5912
5930
|
panels: z.ZodNumber;
|
|
@@ -7300,6 +7318,7 @@ declare const crawlReportSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
7300
7318
|
"robots-blocked": "robots-blocked";
|
|
7301
7319
|
"canonical-conflict": "canonical-conflict";
|
|
7302
7320
|
"canonical-hint-other": "canonical-hint-other";
|
|
7321
|
+
"meta-refresh": "meta-refresh";
|
|
7303
7322
|
"not-html": "not-html";
|
|
7304
7323
|
}>>;
|
|
7305
7324
|
extractionStatus: z.ZodOptional<z.ZodEnum<{
|
|
@@ -7351,6 +7370,10 @@ declare const crawlReportSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
7351
7370
|
indicators: z.ZodArray<z.ZodString>;
|
|
7352
7371
|
formActionPath: z.ZodOptional<z.ZodString>;
|
|
7353
7372
|
}, z.core.$strip>>;
|
|
7373
|
+
metaRefresh: z.ZodOptional<z.ZodObject<{
|
|
7374
|
+
targetUrl: z.ZodString;
|
|
7375
|
+
delaySeconds: z.ZodNumber;
|
|
7376
|
+
}, z.core.$strip>>;
|
|
7354
7377
|
tabbedContent: z.ZodOptional<z.ZodObject<{
|
|
7355
7378
|
groups: z.ZodNumber;
|
|
7356
7379
|
panels: z.ZodNumber;
|
|
@@ -8744,6 +8767,7 @@ declare const crawlReportSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
8744
8767
|
"robots-blocked": "robots-blocked";
|
|
8745
8768
|
"canonical-conflict": "canonical-conflict";
|
|
8746
8769
|
"canonical-hint-other": "canonical-hint-other";
|
|
8770
|
+
"meta-refresh": "meta-refresh";
|
|
8747
8771
|
"not-html": "not-html";
|
|
8748
8772
|
}>>;
|
|
8749
8773
|
extractionStatus: z.ZodOptional<z.ZodEnum<{
|
|
@@ -8795,6 +8819,10 @@ declare const crawlReportSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
8795
8819
|
indicators: z.ZodArray<z.ZodString>;
|
|
8796
8820
|
formActionPath: z.ZodOptional<z.ZodString>;
|
|
8797
8821
|
}, z.core.$strip>>;
|
|
8822
|
+
metaRefresh: z.ZodOptional<z.ZodObject<{
|
|
8823
|
+
targetUrl: z.ZodString;
|
|
8824
|
+
delaySeconds: z.ZodNumber;
|
|
8825
|
+
}, z.core.$strip>>;
|
|
8798
8826
|
tabbedContent: z.ZodOptional<z.ZodObject<{
|
|
8799
8827
|
groups: z.ZodNumber;
|
|
8800
8828
|
panels: z.ZodNumber;
|
|
@@ -10190,6 +10218,7 @@ declare const crawlerSchemas: {
|
|
|
10190
10218
|
"robots-blocked": "robots-blocked";
|
|
10191
10219
|
"canonical-conflict": "canonical-conflict";
|
|
10192
10220
|
"canonical-hint-other": "canonical-hint-other";
|
|
10221
|
+
"meta-refresh": "meta-refresh";
|
|
10193
10222
|
"not-html": "not-html";
|
|
10194
10223
|
}>>;
|
|
10195
10224
|
extractionStatus: z.ZodOptional<z.ZodEnum<{
|
|
@@ -10241,6 +10270,10 @@ declare const crawlerSchemas: {
|
|
|
10241
10270
|
indicators: z.ZodArray<z.ZodString>;
|
|
10242
10271
|
formActionPath: z.ZodOptional<z.ZodString>;
|
|
10243
10272
|
}, z.core.$strip>>;
|
|
10273
|
+
metaRefresh: z.ZodOptional<z.ZodObject<{
|
|
10274
|
+
targetUrl: z.ZodString;
|
|
10275
|
+
delaySeconds: z.ZodNumber;
|
|
10276
|
+
}, z.core.$strip>>;
|
|
10244
10277
|
tabbedContent: z.ZodOptional<z.ZodObject<{
|
|
10245
10278
|
groups: z.ZodNumber;
|
|
10246
10279
|
panels: z.ZodNumber;
|
|
@@ -11634,6 +11667,7 @@ declare const crawlerSchemas: {
|
|
|
11634
11667
|
"robots-blocked": "robots-blocked";
|
|
11635
11668
|
"canonical-conflict": "canonical-conflict";
|
|
11636
11669
|
"canonical-hint-other": "canonical-hint-other";
|
|
11670
|
+
"meta-refresh": "meta-refresh";
|
|
11637
11671
|
"not-html": "not-html";
|
|
11638
11672
|
}>>;
|
|
11639
11673
|
extractionStatus: z.ZodOptional<z.ZodEnum<{
|
|
@@ -11685,6 +11719,10 @@ declare const crawlerSchemas: {
|
|
|
11685
11719
|
indicators: z.ZodArray<z.ZodString>;
|
|
11686
11720
|
formActionPath: z.ZodOptional<z.ZodString>;
|
|
11687
11721
|
}, z.core.$strip>>;
|
|
11722
|
+
metaRefresh: z.ZodOptional<z.ZodObject<{
|
|
11723
|
+
targetUrl: z.ZodString;
|
|
11724
|
+
delaySeconds: z.ZodNumber;
|
|
11725
|
+
}, z.core.$strip>>;
|
|
11688
11726
|
tabbedContent: z.ZodOptional<z.ZodObject<{
|
|
11689
11727
|
groups: z.ZodNumber;
|
|
11690
11728
|
panels: z.ZodNumber;
|
|
@@ -13078,6 +13116,7 @@ declare const crawlerSchemas: {
|
|
|
13078
13116
|
"robots-blocked": "robots-blocked";
|
|
13079
13117
|
"canonical-conflict": "canonical-conflict";
|
|
13080
13118
|
"canonical-hint-other": "canonical-hint-other";
|
|
13119
|
+
"meta-refresh": "meta-refresh";
|
|
13081
13120
|
"not-html": "not-html";
|
|
13082
13121
|
}>>;
|
|
13083
13122
|
extractionStatus: z.ZodOptional<z.ZodEnum<{
|
|
@@ -13129,6 +13168,10 @@ declare const crawlerSchemas: {
|
|
|
13129
13168
|
indicators: z.ZodArray<z.ZodString>;
|
|
13130
13169
|
formActionPath: z.ZodOptional<z.ZodString>;
|
|
13131
13170
|
}, z.core.$strip>>;
|
|
13171
|
+
metaRefresh: z.ZodOptional<z.ZodObject<{
|
|
13172
|
+
targetUrl: z.ZodString;
|
|
13173
|
+
delaySeconds: z.ZodNumber;
|
|
13174
|
+
}, z.core.$strip>>;
|
|
13132
13175
|
tabbedContent: z.ZodOptional<z.ZodObject<{
|
|
13133
13176
|
groups: z.ZodNumber;
|
|
13134
13177
|
panels: z.ZodNumber;
|
|
@@ -14656,6 +14699,7 @@ declare const crawlerSchemas: {
|
|
|
14656
14699
|
"robots-blocked": "robots-blocked";
|
|
14657
14700
|
"canonical-conflict": "canonical-conflict";
|
|
14658
14701
|
"canonical-hint-other": "canonical-hint-other";
|
|
14702
|
+
"meta-refresh": "meta-refresh";
|
|
14659
14703
|
"not-html": "not-html";
|
|
14660
14704
|
}>>;
|
|
14661
14705
|
extractionStatus: z.ZodOptional<z.ZodEnum<{
|
|
@@ -14707,6 +14751,10 @@ declare const crawlerSchemas: {
|
|
|
14707
14751
|
indicators: z.ZodArray<z.ZodString>;
|
|
14708
14752
|
formActionPath: z.ZodOptional<z.ZodString>;
|
|
14709
14753
|
}, z.core.$strip>>;
|
|
14754
|
+
metaRefresh: z.ZodOptional<z.ZodObject<{
|
|
14755
|
+
targetUrl: z.ZodString;
|
|
14756
|
+
delaySeconds: z.ZodNumber;
|
|
14757
|
+
}, z.core.$strip>>;
|
|
14710
14758
|
tabbedContent: z.ZodOptional<z.ZodObject<{
|
|
14711
14759
|
groups: z.ZodNumber;
|
|
14712
14760
|
panels: z.ZodNumber;
|
|
@@ -15435,6 +15483,7 @@ declare const crawlerJsonSchemas: {
|
|
|
15435
15483
|
"robots-blocked": "robots-blocked";
|
|
15436
15484
|
"canonical-conflict": "canonical-conflict";
|
|
15437
15485
|
"canonical-hint-other": "canonical-hint-other";
|
|
15486
|
+
"meta-refresh": "meta-refresh";
|
|
15438
15487
|
"not-html": "not-html";
|
|
15439
15488
|
}>>;
|
|
15440
15489
|
extractionStatus: z.ZodOptional<z.ZodEnum<{
|
|
@@ -15486,6 +15535,10 @@ declare const crawlerJsonSchemas: {
|
|
|
15486
15535
|
indicators: z.ZodArray<z.ZodString>;
|
|
15487
15536
|
formActionPath: z.ZodOptional<z.ZodString>;
|
|
15488
15537
|
}, z.core.$strip>>;
|
|
15538
|
+
metaRefresh: z.ZodOptional<z.ZodObject<{
|
|
15539
|
+
targetUrl: z.ZodString;
|
|
15540
|
+
delaySeconds: z.ZodNumber;
|
|
15541
|
+
}, z.core.$strip>>;
|
|
15489
15542
|
tabbedContent: z.ZodOptional<z.ZodObject<{
|
|
15490
15543
|
groups: z.ZodNumber;
|
|
15491
15544
|
panels: z.ZodNumber;
|
|
@@ -16879,6 +16932,7 @@ declare const crawlerJsonSchemas: {
|
|
|
16879
16932
|
"robots-blocked": "robots-blocked";
|
|
16880
16933
|
"canonical-conflict": "canonical-conflict";
|
|
16881
16934
|
"canonical-hint-other": "canonical-hint-other";
|
|
16935
|
+
"meta-refresh": "meta-refresh";
|
|
16882
16936
|
"not-html": "not-html";
|
|
16883
16937
|
}>>;
|
|
16884
16938
|
extractionStatus: z.ZodOptional<z.ZodEnum<{
|
|
@@ -16930,6 +16984,10 @@ declare const crawlerJsonSchemas: {
|
|
|
16930
16984
|
indicators: z.ZodArray<z.ZodString>;
|
|
16931
16985
|
formActionPath: z.ZodOptional<z.ZodString>;
|
|
16932
16986
|
}, z.core.$strip>>;
|
|
16987
|
+
metaRefresh: z.ZodOptional<z.ZodObject<{
|
|
16988
|
+
targetUrl: z.ZodString;
|
|
16989
|
+
delaySeconds: z.ZodNumber;
|
|
16990
|
+
}, z.core.$strip>>;
|
|
16933
16991
|
tabbedContent: z.ZodOptional<z.ZodObject<{
|
|
16934
16992
|
groups: z.ZodNumber;
|
|
16935
16993
|
panels: z.ZodNumber;
|
|
@@ -18323,6 +18381,7 @@ declare const crawlerJsonSchemas: {
|
|
|
18323
18381
|
"robots-blocked": "robots-blocked";
|
|
18324
18382
|
"canonical-conflict": "canonical-conflict";
|
|
18325
18383
|
"canonical-hint-other": "canonical-hint-other";
|
|
18384
|
+
"meta-refresh": "meta-refresh";
|
|
18326
18385
|
"not-html": "not-html";
|
|
18327
18386
|
}>>;
|
|
18328
18387
|
extractionStatus: z.ZodOptional<z.ZodEnum<{
|
|
@@ -18374,6 +18433,10 @@ declare const crawlerJsonSchemas: {
|
|
|
18374
18433
|
indicators: z.ZodArray<z.ZodString>;
|
|
18375
18434
|
formActionPath: z.ZodOptional<z.ZodString>;
|
|
18376
18435
|
}, z.core.$strip>>;
|
|
18436
|
+
metaRefresh: z.ZodOptional<z.ZodObject<{
|
|
18437
|
+
targetUrl: z.ZodString;
|
|
18438
|
+
delaySeconds: z.ZodNumber;
|
|
18439
|
+
}, z.core.$strip>>;
|
|
18377
18440
|
tabbedContent: z.ZodOptional<z.ZodObject<{
|
|
18378
18441
|
groups: z.ZodNumber;
|
|
18379
18442
|
panels: z.ZodNumber;
|
|
@@ -19901,6 +19964,7 @@ declare const crawlerJsonSchemas: {
|
|
|
19901
19964
|
"robots-blocked": "robots-blocked";
|
|
19902
19965
|
"canonical-conflict": "canonical-conflict";
|
|
19903
19966
|
"canonical-hint-other": "canonical-hint-other";
|
|
19967
|
+
"meta-refresh": "meta-refresh";
|
|
19904
19968
|
"not-html": "not-html";
|
|
19905
19969
|
}>>;
|
|
19906
19970
|
extractionStatus: z.ZodOptional<z.ZodEnum<{
|
|
@@ -19952,6 +20016,10 @@ declare const crawlerJsonSchemas: {
|
|
|
19952
20016
|
indicators: z.ZodArray<z.ZodString>;
|
|
19953
20017
|
formActionPath: z.ZodOptional<z.ZodString>;
|
|
19954
20018
|
}, z.core.$strip>>;
|
|
20019
|
+
metaRefresh: z.ZodOptional<z.ZodObject<{
|
|
20020
|
+
targetUrl: z.ZodString;
|
|
20021
|
+
delaySeconds: z.ZodNumber;
|
|
20022
|
+
}, z.core.$strip>>;
|
|
19955
20023
|
tabbedContent: z.ZodOptional<z.ZodObject<{
|
|
19956
20024
|
groups: z.ZodNumber;
|
|
19957
20025
|
panels: z.ZodNumber;
|
package/dist/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
|
|
2
|
-
import { n as load, t as version } from "./package-
|
|
2
|
+
import { n as load, t as version } from "./package-Cp8DwqCy.js";
|
|
3
3
|
import { Headers, fetch as fetch$1 } from "undici";
|
|
4
4
|
import { chmodSync, createReadStream, createWriteStream, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
5
5
|
import { arch, homedir, platform, totalmem } from "node:os";
|
|
@@ -3176,6 +3176,22 @@ function extractStructuredData($, baseUrl, options = {}) {
|
|
|
3176
3176
|
}
|
|
3177
3177
|
//#endregion
|
|
3178
3178
|
//#region packages/core/src/extract/page-extractor.ts
|
|
3179
|
+
function metaRefresh($, finalUrl) {
|
|
3180
|
+
for (const element of $("meta[http-equiv]").toArray()) {
|
|
3181
|
+
if ($(element).attr("http-equiv")?.trim().toLowerCase() !== "refresh") continue;
|
|
3182
|
+
const match = ($(element).attr("content")?.trim())?.match(/^([0-9]+(?:\.[0-9]+)?)\s*;\s*url\s*=\s*(?:"([^"]*)"|'([^']*)'|(.+))$/iu);
|
|
3183
|
+
if (!match) continue;
|
|
3184
|
+
try {
|
|
3185
|
+
const target = new URL((match[2] ?? match[3] ?? match[4] ?? "").trim(), finalUrl);
|
|
3186
|
+
if (!["http:", "https:"].includes(target.protocol)) continue;
|
|
3187
|
+
target.hash = "";
|
|
3188
|
+
return {
|
|
3189
|
+
targetUrl: target.toString(),
|
|
3190
|
+
delaySeconds: Number(match[1])
|
|
3191
|
+
};
|
|
3192
|
+
} catch {}
|
|
3193
|
+
}
|
|
3194
|
+
}
|
|
3179
3195
|
const SOCIAL_PROFILE_HOSTS = [
|
|
3180
3196
|
"facebook.com",
|
|
3181
3197
|
"instagram.com",
|
|
@@ -3455,6 +3471,8 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
|
|
|
3455
3471
|
}, extractor, dependencies);
|
|
3456
3472
|
const { text, excerpt } = content;
|
|
3457
3473
|
const title = safeText$1($("title").first().text());
|
|
3474
|
+
const crawlerMetaRefresh = extractor === "crawler" ? metaRefresh($, fetchResult.finalUrl) : void 0;
|
|
3475
|
+
if (crawlerMetaRefresh && new URL(crawlerMetaRefresh.targetUrl).origin === url.origin) crawlerInternalLinks.add(crawlerMetaRefresh.targetUrl);
|
|
3458
3476
|
return {
|
|
3459
3477
|
url: fetchResult.url,
|
|
3460
3478
|
finalUrl: fetchResult.finalUrl,
|
|
@@ -3480,6 +3498,7 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
|
|
|
3480
3498
|
externalAnchorSamples,
|
|
3481
3499
|
socialProfileLinks: [...socialProfileLinks],
|
|
3482
3500
|
contentSketch: contentSketch(text),
|
|
3501
|
+
metaRefresh: crawlerMetaRefresh,
|
|
3483
3502
|
softAuthenticationGate: softAuthenticationGate($, fetchResult.finalUrl, title),
|
|
3484
3503
|
tabbedContent: tabbedContent($)
|
|
3485
3504
|
} } : {},
|
|
@@ -8289,6 +8308,9 @@ function failedMarkdownUrls(discovery) {
|
|
|
8289
8308
|
return !negotiatedOk || !explicitOk || page.repeatedHashStable !== true;
|
|
8290
8309
|
}).map((page) => page.htmlUrl);
|
|
8291
8310
|
}
|
|
8311
|
+
function unstableMarkdownUrls(discovery) {
|
|
8312
|
+
return discovery.markdownAlternates.pages.filter((page) => confirmedMarkdown(page) !== void 0 && page.repeatedHashStable !== true).map((page) => page.htmlUrl);
|
|
8313
|
+
}
|
|
8292
8314
|
function qualityFailureUrls(discovery) {
|
|
8293
8315
|
return discovery.markdownAlternates.pages.filter((page) => {
|
|
8294
8316
|
const quality = page.quality;
|
|
@@ -8302,7 +8324,10 @@ function confirmedMarkdown(page) {
|
|
|
8302
8324
|
function markdownChecks(discovery) {
|
|
8303
8325
|
const markdown = discovery.markdownAlternates;
|
|
8304
8326
|
const failedUrls = failedMarkdownUrls(discovery);
|
|
8327
|
+
const unstableUrls = unstableMarkdownUrls(discovery);
|
|
8305
8328
|
const qualityUrls = qualityFailureUrls(discovery);
|
|
8329
|
+
const hasEligibleHtml = markdown.eligibleHtmlPages > 0;
|
|
8330
|
+
const hasEvaluatedMarkdown = markdown.evaluatedPages > 0;
|
|
8306
8331
|
const negotiatedPages = markdown.pages.filter((page) => page.negotiated?.status === 200 && /^\s*text\/markdown\b/iu.test(page.negotiated.contentType ?? "")).length;
|
|
8307
8332
|
const explicitPages = markdown.pages.filter((page) => page.htmlAlternateUnique).length;
|
|
8308
8333
|
const coverageComplete = markdown.eligibleHtmlPages > 0 && markdown.evaluatedPages === markdown.eligibleHtmlPages;
|
|
@@ -8331,25 +8356,28 @@ function markdownChecks(discovery) {
|
|
|
8331
8356
|
check$2("representations", {
|
|
8332
8357
|
id: "markdown-coverage",
|
|
8333
8358
|
status: markdown.eligibleHtmlPages === 0 ? "unknown" : coverageComplete ? "pass" : "fail",
|
|
8334
|
-
title: coverageComplete ? "Every successful HTML page has a Markdown representation" : "Markdown representation coverage is incomplete",
|
|
8335
|
-
plainEnglish: `${markdown.evaluatedPages} of ${markdown.eligibleHtmlPages} successful HTML pages returned Markdown. ${explicitPages} advertised an explicit alternative and ${negotiatedPages} supported content negotiation
|
|
8336
|
-
action: coverageComplete ? "Keep the negotiated or explicit Markdown response stable as routes change." : "Return text/markdown through content negotiation or advertise one working Markdown alternative for each public HTML page.",
|
|
8359
|
+
title: coverageComplete ? "Every successful HTML page has a Markdown representation" : !hasEligibleHtml ? "Markdown representation coverage was not evaluated" : "Markdown representation coverage is incomplete",
|
|
8360
|
+
plainEnglish: hasEligibleHtml ? `${markdown.evaluatedPages} of ${markdown.eligibleHtmlPages} successful HTML pages returned Markdown. ${explicitPages} advertised an explicit alternative and ${negotiatedPages} supported content negotiation.` : "No successful HTML page was available, so Markdown representation coverage could not be evaluated.",
|
|
8361
|
+
action: !hasEligibleHtml ? "Resolve the crawl or page-fetch failure, then run this check again." : coverageComplete ? "Keep the negotiated or explicit Markdown response stable as routes change." : "Return text/markdown through content negotiation or advertise one working Markdown alternative for each public HTML page.",
|
|
8337
8362
|
evidence: {
|
|
8338
8363
|
eligibleHtmlPages: markdown.eligibleHtmlPages,
|
|
8339
8364
|
advertisedPages: markdown.advertisedPages,
|
|
8340
8365
|
negotiatedPages,
|
|
8341
8366
|
evaluatedPages: markdown.evaluatedPages
|
|
8342
8367
|
},
|
|
8343
|
-
urls: failedUrls.slice(0, 25)
|
|
8368
|
+
urls: hasEligibleHtml ? failedUrls.slice(0, 25) : []
|
|
8344
8369
|
}),
|
|
8345
8370
|
check$2("representations", {
|
|
8346
8371
|
id: "markdown-token-estimates",
|
|
8347
8372
|
status: markdown.evaluatedPages === 0 ? "unknown" : tokenHeaders === markdown.evaluatedPages ? "pass" : "info",
|
|
8348
|
-
title: tokenHeaders === markdown.evaluatedPages ? "Every Markdown response includes a token estimate" : "Some Markdown responses are missing a stable token estimate",
|
|
8349
|
-
plainEnglish: `${tokenHeaders} of ${markdown.evaluatedPages} evaluated pages returned X-Markdown-Tokens. Paired explicit and negotiated responses agreed where both existed. The value is an estimate, not an exact model-specific token count
|
|
8350
|
-
action: tokenHeaders === markdown.evaluatedPages ? "Keep the estimate stable when both explicit and negotiated responses exist." : "No change is required. X-Markdown-Tokens is optional; local size estimates are reported separately.",
|
|
8373
|
+
title: !hasEvaluatedMarkdown ? "Markdown token estimates were not evaluated" : tokenHeaders === markdown.evaluatedPages ? "Every Markdown response includes a token estimate" : "Some Markdown responses are missing a stable token estimate",
|
|
8374
|
+
plainEnglish: hasEvaluatedMarkdown ? `${tokenHeaders} of ${markdown.evaluatedPages} evaluated pages returned X-Markdown-Tokens. Paired explicit and negotiated responses agreed where both existed. The value is an estimate, not an exact model-specific token count.` : "No confirmed Markdown response was available, so optional server token estimates could not be evaluated.",
|
|
8375
|
+
action: !hasEvaluatedMarkdown ? "Confirm a successful text/markdown representation, then run this check again." : tokenHeaders === markdown.evaluatedPages ? "Keep the estimate stable when both explicit and negotiated responses exist." : "No change is required. X-Markdown-Tokens is optional; local size estimates are reported separately.",
|
|
8351
8376
|
urls: markdown.pages.filter((page) => {
|
|
8352
|
-
|
|
8377
|
+
const primary = confirmedMarkdown(page);
|
|
8378
|
+
if (!primary) return false;
|
|
8379
|
+
const paired = [page.explicit, page.negotiated].filter((response) => response?.status !== void 0 && response.status >= 200 && response.status < 300 && /^\s*text\/markdown\b/iu.test(response.contentType ?? ""));
|
|
8380
|
+
return primary?.markdownTokens === void 0 || paired.length === 2 && paired[0]?.markdownTokens !== paired[1]?.markdownTokens;
|
|
8353
8381
|
}).map((page) => page.htmlUrl).slice(0, 25)
|
|
8354
8382
|
}),
|
|
8355
8383
|
check$2("representations", {
|
|
@@ -8357,7 +8385,7 @@ function markdownChecks(discovery) {
|
|
|
8357
8385
|
status: sizedPages.length === 0 ? "unknown" : largePages.length === 0 ? "pass" : "warning",
|
|
8358
8386
|
title: sizedPages.length === 0 ? "Markdown size was not measured" : largePages.length === 0 ? "Markdown representations stay within the review threshold" : "Some Markdown representations are large enough to review",
|
|
8359
8387
|
plainEnglish: `${sizedPages.length} Markdown responses were measured locally. ${largePages.length} exceeded 50,000 characters and ${veryLargePages.length} exceeded 100,000 characters. These are processing-risk thresholds, not universal model limits.`,
|
|
8360
|
-
action: largePages.length === 0 ? "Keep important content early and retain stable section structure as pages grow." : "Split genuinely separate topics, keep navigation and repeated chrome out of Markdown, and put the most useful content early. Do not remove necessary detail just to hit a threshold.",
|
|
8388
|
+
action: sizedPages.length === 0 ? "Confirm a successful text/markdown representation, then run this check again." : largePages.length === 0 ? "Keep important content early and retain stable section structure as pages grow." : "Split genuinely separate topics, keep navigation and repeated chrome out of Markdown, and put the most useful content early. Do not remove necessary detail just to hit a threshold.",
|
|
8361
8389
|
evidence: {
|
|
8362
8390
|
measuredPages: sizedPages.length,
|
|
8363
8391
|
reviewThresholdCharacters: 5e4,
|
|
@@ -8375,7 +8403,7 @@ function markdownChecks(discovery) {
|
|
|
8375
8403
|
status: parityPages.length === 0 ? "unknown" : parityFailures.length === 0 ? "pass" : "warning",
|
|
8376
8404
|
title: parityPages.length === 0 ? "HTML and Markdown content parity was not measured" : parityFailures.length === 0 ? "Markdown retains sampled content from the HTML document" : "Some Markdown responses may omit important HTML content",
|
|
8377
8405
|
plainEnglish: `${parityPages.length} pages were compared using bounded hashes sampled across the main HTML content. ${parityFailures.length} retained less than 60% of that sample. This detects likely omissions without storing full page bodies.`,
|
|
8378
|
-
action: parityFailures.length === 0 ? "Keep the shared conversion path covered as templates and components change." : "Compare the affected HTML and Markdown around the missing sections, then fix the shared converter or component serialization.",
|
|
8406
|
+
action: parityPages.length === 0 ? "Confirm a successful text/markdown representation, then run this check again." : parityFailures.length === 0 ? "Keep the shared conversion path covered as templates and components change." : "Compare the affected HTML and Markdown around the missing sections, then fix the shared converter or component serialization.",
|
|
8379
8407
|
evidence: {
|
|
8380
8408
|
evaluatedPages: parityPages.length,
|
|
8381
8409
|
failedPages: parityFailures.length,
|
|
@@ -8400,32 +8428,32 @@ function markdownChecks(discovery) {
|
|
|
8400
8428
|
check$2("representations", {
|
|
8401
8429
|
id: "markdown-negotiation",
|
|
8402
8430
|
status: markdown.eligibleHtmlPages === 0 ? "unknown" : negotiationComplete && discovery.contentNegotiation.qZeroHonoured ? "pass" : "fail",
|
|
8403
|
-
title: negotiationComplete && discovery.contentNegotiation.qZeroHonoured ? "Markdown content negotiation works across the site" : "Markdown content negotiation needs attention",
|
|
8404
|
-
plainEnglish: `${negotiatedPages} of ${markdown.eligibleHtmlPages} pages returned negotiated Markdown. ${markdown.exactByteMatches} of ${explicitPages} paired explicit responses matched byte for byte. A request that refuses Markdown ${discovery.contentNegotiation.qZeroHonoured ? "received HTML" : "did not produce confirmed HTML evidence"}
|
|
8405
|
-
action: "Honour Accept q-values, send Vary: Accept, and keep paired explicit and negotiated responses identical when both exist.",
|
|
8431
|
+
title: !hasEligibleHtml ? "Markdown content negotiation was not evaluated" : negotiationComplete && discovery.contentNegotiation.qZeroHonoured ? "Markdown content negotiation works across the site" : "Markdown content negotiation needs attention",
|
|
8432
|
+
plainEnglish: hasEligibleHtml ? `${negotiatedPages} of ${markdown.eligibleHtmlPages} pages returned negotiated Markdown. ${markdown.exactByteMatches} of ${explicitPages} paired explicit responses matched byte for byte. A request that refuses Markdown ${discovery.contentNegotiation.qZeroHonoured ? "received HTML" : "did not produce confirmed HTML evidence"}.` : "No successful HTML page was available, so content negotiation could not be evaluated.",
|
|
8433
|
+
action: hasEligibleHtml ? "Honour Accept q-values, send Vary: Accept, and keep paired explicit and negotiated responses identical when both exist." : "Resolve the crawl or page-fetch failure, then run this check again.",
|
|
8406
8434
|
evidence: {
|
|
8407
8435
|
negotiatedPages,
|
|
8408
8436
|
explicitPages,
|
|
8409
8437
|
exactByteMatches: markdown.exactByteMatches,
|
|
8410
8438
|
qZero: discovery.contentNegotiation
|
|
8411
8439
|
},
|
|
8412
|
-
urls: failedUrls.slice(0, 25)
|
|
8440
|
+
urls: hasEligibleHtml ? failedUrls.slice(0, 25) : []
|
|
8413
8441
|
}),
|
|
8414
8442
|
check$2("representations", {
|
|
8415
8443
|
id: "markdown-determinism",
|
|
8416
8444
|
status: markdown.evaluatedPages === 0 ? "unknown" : stable ? "pass" : "fail",
|
|
8417
|
-
title: stable ? "Repeated Markdown responses are stable" : "Repeated Markdown responses changed during the audit",
|
|
8418
|
-
plainEnglish: `${markdown.stableResponses} of ${markdown.evaluatedPages} repeated Markdown responses kept the same SHA-256 digest
|
|
8419
|
-
action: "Remove timestamps, random output, or other runtime rewriting that changes the same Markdown response between requests.",
|
|
8445
|
+
title: !hasEvaluatedMarkdown ? "Markdown response stability was not evaluated" : stable ? "Repeated Markdown responses are stable" : "Repeated Markdown responses changed during the audit",
|
|
8446
|
+
plainEnglish: hasEvaluatedMarkdown ? `${markdown.stableResponses} of ${markdown.evaluatedPages} repeated Markdown responses kept the same SHA-256 digest.` : "No confirmed Markdown response was available for a repeated stability check.",
|
|
8447
|
+
action: !hasEvaluatedMarkdown ? "Confirm a successful text/markdown representation, then run this check again." : stable ? "Keep repeated Markdown responses byte-stable as templates and data change." : "Remove timestamps, random output, or other runtime rewriting that changes the same Markdown response between requests.",
|
|
8420
8448
|
evidence: { stableResponses: markdown.stableResponses },
|
|
8421
|
-
urls:
|
|
8449
|
+
urls: unstableUrls.slice(0, 25)
|
|
8422
8450
|
}),
|
|
8423
8451
|
check$2("representations", {
|
|
8424
8452
|
id: "markdown-quality",
|
|
8425
8453
|
status: markdown.evaluatedPages === 0 ? "unknown" : qualityUrls.length === 0 ? "pass" : "warning",
|
|
8426
|
-
title: qualityUrls.length === 0 ? "Markdown keeps the useful document structure cleanly" : "Some Markdown alternatives need an extraction review",
|
|
8427
|
-
plainEnglish: qualityUrls.length === 0 ? "The evaluated alternatives retained one H1, frontmatter titles, balanced code fences, useful copy, and no leaked SVG, script, style, or layout tags." : `${qualityUrls.length} Markdown alternative${qualityUrls.length === 1 ? "" : "s"} lost important structure, looked navigation-only, or leaked presentation markup.`,
|
|
8428
|
-
action: "Open the affected Markdown directly and fix the shared converter or component exclusion rules rather than patching generated files.",
|
|
8454
|
+
title: !hasEvaluatedMarkdown ? "Markdown quality was not evaluated" : qualityUrls.length === 0 ? "Markdown keeps the useful document structure cleanly" : "Some Markdown alternatives need an extraction review",
|
|
8455
|
+
plainEnglish: !hasEvaluatedMarkdown ? "No confirmed Markdown response was available for structure and extraction checks." : qualityUrls.length === 0 ? "The evaluated alternatives retained one H1, frontmatter titles, balanced code fences, useful copy, and no leaked SVG, script, style, or layout tags." : `${qualityUrls.length} Markdown alternative${qualityUrls.length === 1 ? "" : "s"} lost important structure, looked navigation-only, or leaked presentation markup.`,
|
|
8456
|
+
action: !hasEvaluatedMarkdown ? "Confirm a successful text/markdown representation, then run this check again." : qualityUrls.length === 0 ? "Keep the shared Markdown conversion path covered as templates and components change." : "Open the affected Markdown directly and fix the shared converter or component exclusion rules rather than patching generated files.",
|
|
8429
8457
|
urls: qualityUrls.slice(0, 25)
|
|
8430
8458
|
})
|
|
8431
8459
|
];
|
|
@@ -10349,7 +10377,8 @@ const RULE_DEFINITIONS = [
|
|
|
10349
10377
|
evidenceFields: [
|
|
10350
10378
|
"page.url",
|
|
10351
10379
|
"page.finalUrl",
|
|
10352
|
-
"page.status"
|
|
10380
|
+
"page.status",
|
|
10381
|
+
"page.metaRefresh"
|
|
10353
10382
|
],
|
|
10354
10383
|
suggestedCommands: ["seo redirect-trace --url <url> --json"]
|
|
10355
10384
|
}
|
|
@@ -10846,6 +10875,7 @@ function comparableUrl(value) {
|
|
|
10846
10875
|
}
|
|
10847
10876
|
}
|
|
10848
10877
|
function isRedirectedPage(page) {
|
|
10878
|
+
if (page.metaRefresh) return true;
|
|
10849
10879
|
const requested = comparableUrl(page.url);
|
|
10850
10880
|
const final = comparableUrl(page.finalUrl);
|
|
10851
10881
|
return Boolean(requested && final && requested !== final);
|
|
@@ -11120,11 +11150,18 @@ function auditCrawlPages(pages, opts = {}) {
|
|
|
11120
11150
|
}
|
|
11121
11151
|
if (page.status < 200) continue;
|
|
11122
11152
|
const redirected = isRedirectedPage(page);
|
|
11123
|
-
if (redirected)
|
|
11124
|
-
|
|
11125
|
-
|
|
11126
|
-
|
|
11127
|
-
|
|
11153
|
+
if (redirected) {
|
|
11154
|
+
const redirectTarget = page.metaRefresh?.targetUrl ?? page.finalUrl;
|
|
11155
|
+
issues.push(issue("redirected_url", page, `Final URL: ${redirectTarget}`, {
|
|
11156
|
+
requestedUrl: page.url,
|
|
11157
|
+
finalUrl: redirectTarget,
|
|
11158
|
+
status: page.status,
|
|
11159
|
+
...page.metaRefresh ? {
|
|
11160
|
+
redirectType: "meta-refresh",
|
|
11161
|
+
delaySeconds: page.metaRefresh.delaySeconds
|
|
11162
|
+
} : {}
|
|
11163
|
+
}));
|
|
11164
|
+
}
|
|
11128
11165
|
const redirectChain = page.fetchDiagnostics?.redirectChain ?? [];
|
|
11129
11166
|
if (redirectChain.length > 1) issues.push(issue("redirect_chain", page, `${redirectChain.length} hops`, {
|
|
11130
11167
|
redirectChain,
|
|
@@ -11980,7 +12017,7 @@ function latestCrawlReport(site, store = crawlReportStore) {
|
|
|
11980
12017
|
//#region packages/core/src/analyze/crawler/crawl-caveats.ts
|
|
11981
12018
|
function crawlCaveats(input) {
|
|
11982
12019
|
const caveats = input.cancelled ? ["Crawl cancelled before all queued URLs finished."] : input.pageLimitReached ? [`Stopped after reaching maxPages (${input.maxPages}).`] : [];
|
|
11983
|
-
if (input.queueSafetySkippedUrls > 0) caveats.push(`Left ${input.queueSafetySkippedUrls} eligible same-origin URLs unqueued to keep this crawl bounded. Increase --max-pages to inspect more.`);
|
|
12020
|
+
if (input.queueSafetySkippedUrls > 0) caveats.push(input.queueSafetyInventoryCapped ? `Left at least ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. The excluded-URL inventory reached its reporting cap. Increase --max-pages to inspect more.` : `Left ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. Increase --max-pages to inspect more.`);
|
|
11984
12021
|
if (input.originBackpressureSkippedUrls > 0) caveats.push(`Stopped ${input.originBackpressureSkippedUrls} queued URL${input.originBackpressureSkippedUrls === 1 ? "" : "s"} because the origin stayed slow. This is incomplete crawl evidence, not a failed URL check.`);
|
|
11985
12022
|
if (input.memoryPressureSkippedUrls > 0) caveats.push(`Left ${input.memoryPressureSkippedUrls} eligible URL${input.memoryPressureSkippedUrls === 1 ? "" : "s"} unchecked after the local memory safety limit was reached. Start with --health or lower --max-pages.`);
|
|
11986
12023
|
return caveats;
|
|
@@ -13133,6 +13170,7 @@ async function crawlOne(url, opts = {}) {
|
|
|
13133
13170
|
contentSample: truncate$1(extracted.contentText, 300),
|
|
13134
13171
|
contentSketch: crawlerEvidence.contentSketch,
|
|
13135
13172
|
softAuthenticationGate: crawlerEvidence.softAuthenticationGate,
|
|
13173
|
+
metaRefresh: crawlerEvidence.metaRefresh,
|
|
13136
13174
|
tabbedContent: crawlerEvidence.tabbedContent,
|
|
13137
13175
|
lang: extracted.lang,
|
|
13138
13176
|
hasViewport: extracted.hasViewport,
|
|
@@ -13183,9 +13221,9 @@ async function crawlOne(url, opts = {}) {
|
|
|
13183
13221
|
hasQapageSchema: extracted.schemaTypes.includes("QAPage")
|
|
13184
13222
|
}
|
|
13185
13223
|
};
|
|
13186
|
-
page.indexability = indexabilityReason(page);
|
|
13224
|
+
page.indexability = page.metaRefresh ? `Meta refresh to ${page.metaRefresh.targetUrl}` : indexabilityReason(page);
|
|
13187
13225
|
page.indexable = !page.indexability;
|
|
13188
|
-
page.declaredIndexability = hasMetaRobotsDirective(page.metaRobots, "noindex") || hasXRobotsDirective(page.xRobotsTag, "noindex") ? "noindex" : page.robotsTxt?.allowed === false ? "robots-blocked" : page.canonicalStatus === "conflicting" ? "canonical-conflict" : page.canonical && !sameDocumentUrl(page.canonical, page.finalUrl) ? "canonical-hint-other" : "indexable-candidate";
|
|
13226
|
+
page.declaredIndexability = page.metaRefresh ? "meta-refresh" : hasMetaRobotsDirective(page.metaRobots, "noindex") || hasXRobotsDirective(page.xRobotsTag, "noindex") ? "noindex" : page.robotsTxt?.allowed === false ? "robots-blocked" : page.canonicalStatus === "conflicting" ? "canonical-conflict" : page.canonical && !sameDocumentUrl(page.canonical, page.finalUrl) ? "canonical-hint-other" : "indexable-candidate";
|
|
13189
13227
|
return {
|
|
13190
13228
|
request: responseObservation(url, fetched, "complete"),
|
|
13191
13229
|
page,
|
|
@@ -13979,6 +14017,9 @@ async function crawlSite(input, dependencies) {
|
|
|
13979
14017
|
let robotsDeferredUrls = 0;
|
|
13980
14018
|
let originBackpressureSkippedUrls = 0;
|
|
13981
14019
|
let queueSafetySkippedUrls = 0;
|
|
14020
|
+
const queueSafetyDeferredUrls = /* @__PURE__ */ new Set();
|
|
14021
|
+
const maxTrackedQueueSafetyUrls = Math.min(5e4, Math.max(1e3, config.maxPages * 5));
|
|
14022
|
+
let queueSafetyInventoryCapped = false;
|
|
13982
14023
|
let failedUrls = 0;
|
|
13983
14024
|
let observedInternalLinks = 0;
|
|
13984
14025
|
let memoryLimited = false;
|
|
@@ -14083,13 +14124,16 @@ async function crawlSite(input, dependencies) {
|
|
|
14083
14124
|
return;
|
|
14084
14125
|
}
|
|
14085
14126
|
if (queue.size + visited.size + inFlight.size >= config.maxPages * 5) {
|
|
14086
|
-
|
|
14087
|
-
|
|
14088
|
-
|
|
14089
|
-
|
|
14090
|
-
|
|
14091
|
-
|
|
14092
|
-
|
|
14127
|
+
if (!queueSafetyDeferredUrls.has(normalized)) if (queueSafetyDeferredUrls.size < maxTrackedQueueSafetyUrls) {
|
|
14128
|
+
queueSafetyDeferredUrls.add(normalized);
|
|
14129
|
+
queueSafetySkippedUrls = queueSafetyDeferredUrls.size;
|
|
14130
|
+
recordSkip("queue-safety-limit");
|
|
14131
|
+
emitStatus("url_skipped", {
|
|
14132
|
+
url: normalized,
|
|
14133
|
+
depth,
|
|
14134
|
+
reason: "queue_safety_limit"
|
|
14135
|
+
});
|
|
14136
|
+
} else queueSafetyInventoryCapped = true;
|
|
14093
14137
|
return;
|
|
14094
14138
|
}
|
|
14095
14139
|
discovered.add(normalized);
|
|
@@ -14462,6 +14506,7 @@ async function crawlSite(input, dependencies) {
|
|
|
14462
14506
|
pageLimitReached,
|
|
14463
14507
|
maxPages: config.maxPages,
|
|
14464
14508
|
queueSafetySkippedUrls,
|
|
14509
|
+
queueSafetyInventoryCapped,
|
|
14465
14510
|
originBackpressureSkippedUrls,
|
|
14466
14511
|
memoryPressureSkippedUrls
|
|
14467
14512
|
}),
|
|
@@ -17356,6 +17401,7 @@ const crawlPageSnapshotSchema = z.object({
|
|
|
17356
17401
|
"robots-blocked",
|
|
17357
17402
|
"canonical-conflict",
|
|
17358
17403
|
"canonical-hint-other",
|
|
17404
|
+
"meta-refresh",
|
|
17359
17405
|
"not-html",
|
|
17360
17406
|
"unknown"
|
|
17361
17407
|
]).optional(),
|
|
@@ -17402,6 +17448,10 @@ const crawlPageSnapshotSchema = z.object({
|
|
|
17402
17448
|
indicators: z.array(z.string()),
|
|
17403
17449
|
formActionPath: z.string().optional()
|
|
17404
17450
|
}).optional(),
|
|
17451
|
+
metaRefresh: z.object({
|
|
17452
|
+
targetUrl: z.string().url(),
|
|
17453
|
+
delaySeconds: z.number().nonnegative()
|
|
17454
|
+
}).optional(),
|
|
17405
17455
|
tabbedContent: z.object({
|
|
17406
17456
|
groups: z.number().int().positive(),
|
|
17407
17457
|
panels: z.number().int().positive(),
|