seo 0.2.13 → 0.2.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -980,7 +980,7 @@ declare const RULE_DEFINITIONS: readonly [{
980
980
  readonly impactIfIgnored: "An avoidable redirect adds a request and can turn into a chain later. An intentional direct redirect needs no change.";
981
981
  readonly howToVerify: "Re-run the crawl and confirm the requested URL is the same as the final URL, or that the redirect is intentional.";
982
982
  readonly agentHints: {
983
- readonly evidenceFields: ["page.url", "page.finalUrl", "page.status"];
983
+ readonly evidenceFields: ["page.url", "page.finalUrl", "page.status", "page.metaRefresh"];
984
984
  readonly suggestedCommands: ["seo redirect-trace --url <url> --json"];
985
985
  };
986
986
  }, {
@@ -2012,6 +2012,10 @@ type CrawlerExtractionEvidence = {
2012
2012
  }>;
2013
2013
  socialProfileLinks: string[];
2014
2014
  contentSketch: ContentSketch;
2015
+ metaRefresh?: {
2016
+ targetUrl: string;
2017
+ delaySeconds: number;
2018
+ };
2015
2019
  softAuthenticationGate?: {
2016
2020
  kind: 'login-form';
2017
2021
  indicators: string[];
@@ -2495,7 +2499,7 @@ type CrawlPageSnapshot = {
2495
2499
  h3Count?: number;
2496
2500
  indexable: boolean;
2497
2501
  indexability?: string;
2498
- declaredIndexability?: 'indexable-candidate' | 'noindex' | 'robots-blocked' | 'canonical-conflict' | 'canonical-hint-other' | 'not-html' | 'unknown';
2502
+ declaredIndexability?: 'indexable-candidate' | 'noindex' | 'robots-blocked' | 'canonical-conflict' | 'canonical-hint-other' | 'meta-refresh' | 'not-html' | 'unknown';
2499
2503
  extractionStatus?: 'complete' | 'failed' | 'not-applicable' | 'unknown-media-type';
2500
2504
  extractionError?: string;
2501
2505
  wordCount: number;
@@ -2511,6 +2515,10 @@ type CrawlPageSnapshot = {
2511
2515
  indicators: string[];
2512
2516
  formActionPath?: string;
2513
2517
  };
2518
+ metaRefresh?: {
2519
+ targetUrl: string;
2520
+ delaySeconds: number;
2521
+ };
2514
2522
  tabbedContent?: {
2515
2523
  groups: number;
2516
2524
  panels: number;
@@ -4697,6 +4705,7 @@ declare const crawlPageSnapshotSchema: z.ZodObject<{
4697
4705
  "robots-blocked": "robots-blocked";
4698
4706
  "canonical-conflict": "canonical-conflict";
4699
4707
  "canonical-hint-other": "canonical-hint-other";
4708
+ "meta-refresh": "meta-refresh";
4700
4709
  "not-html": "not-html";
4701
4710
  }>>;
4702
4711
  extractionStatus: z.ZodOptional<z.ZodEnum<{
@@ -4748,6 +4757,10 @@ declare const crawlPageSnapshotSchema: z.ZodObject<{
4748
4757
  indicators: z.ZodArray<z.ZodString>;
4749
4758
  formActionPath: z.ZodOptional<z.ZodString>;
4750
4759
  }, z.core.$strip>>;
4760
+ metaRefresh: z.ZodOptional<z.ZodObject<{
4761
+ targetUrl: z.ZodString;
4762
+ delaySeconds: z.ZodNumber;
4763
+ }, z.core.$strip>>;
4751
4764
  tabbedContent: z.ZodOptional<z.ZodObject<{
4752
4765
  groups: z.ZodNumber;
4753
4766
  panels: z.ZodNumber;
@@ -5856,6 +5869,7 @@ declare const crawlReportSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
5856
5869
  "robots-blocked": "robots-blocked";
5857
5870
  "canonical-conflict": "canonical-conflict";
5858
5871
  "canonical-hint-other": "canonical-hint-other";
5872
+ "meta-refresh": "meta-refresh";
5859
5873
  "not-html": "not-html";
5860
5874
  }>>;
5861
5875
  extractionStatus: z.ZodOptional<z.ZodEnum<{
@@ -5907,6 +5921,10 @@ declare const crawlReportSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
5907
5921
  indicators: z.ZodArray<z.ZodString>;
5908
5922
  formActionPath: z.ZodOptional<z.ZodString>;
5909
5923
  }, z.core.$strip>>;
5924
+ metaRefresh: z.ZodOptional<z.ZodObject<{
5925
+ targetUrl: z.ZodString;
5926
+ delaySeconds: z.ZodNumber;
5927
+ }, z.core.$strip>>;
5910
5928
  tabbedContent: z.ZodOptional<z.ZodObject<{
5911
5929
  groups: z.ZodNumber;
5912
5930
  panels: z.ZodNumber;
@@ -7300,6 +7318,7 @@ declare const crawlReportSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
7300
7318
  "robots-blocked": "robots-blocked";
7301
7319
  "canonical-conflict": "canonical-conflict";
7302
7320
  "canonical-hint-other": "canonical-hint-other";
7321
+ "meta-refresh": "meta-refresh";
7303
7322
  "not-html": "not-html";
7304
7323
  }>>;
7305
7324
  extractionStatus: z.ZodOptional<z.ZodEnum<{
@@ -7351,6 +7370,10 @@ declare const crawlReportSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
7351
7370
  indicators: z.ZodArray<z.ZodString>;
7352
7371
  formActionPath: z.ZodOptional<z.ZodString>;
7353
7372
  }, z.core.$strip>>;
7373
+ metaRefresh: z.ZodOptional<z.ZodObject<{
7374
+ targetUrl: z.ZodString;
7375
+ delaySeconds: z.ZodNumber;
7376
+ }, z.core.$strip>>;
7354
7377
  tabbedContent: z.ZodOptional<z.ZodObject<{
7355
7378
  groups: z.ZodNumber;
7356
7379
  panels: z.ZodNumber;
@@ -8744,6 +8767,7 @@ declare const crawlReportSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
8744
8767
  "robots-blocked": "robots-blocked";
8745
8768
  "canonical-conflict": "canonical-conflict";
8746
8769
  "canonical-hint-other": "canonical-hint-other";
8770
+ "meta-refresh": "meta-refresh";
8747
8771
  "not-html": "not-html";
8748
8772
  }>>;
8749
8773
  extractionStatus: z.ZodOptional<z.ZodEnum<{
@@ -8795,6 +8819,10 @@ declare const crawlReportSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
8795
8819
  indicators: z.ZodArray<z.ZodString>;
8796
8820
  formActionPath: z.ZodOptional<z.ZodString>;
8797
8821
  }, z.core.$strip>>;
8822
+ metaRefresh: z.ZodOptional<z.ZodObject<{
8823
+ targetUrl: z.ZodString;
8824
+ delaySeconds: z.ZodNumber;
8825
+ }, z.core.$strip>>;
8798
8826
  tabbedContent: z.ZodOptional<z.ZodObject<{
8799
8827
  groups: z.ZodNumber;
8800
8828
  panels: z.ZodNumber;
@@ -10190,6 +10218,7 @@ declare const crawlerSchemas: {
10190
10218
  "robots-blocked": "robots-blocked";
10191
10219
  "canonical-conflict": "canonical-conflict";
10192
10220
  "canonical-hint-other": "canonical-hint-other";
10221
+ "meta-refresh": "meta-refresh";
10193
10222
  "not-html": "not-html";
10194
10223
  }>>;
10195
10224
  extractionStatus: z.ZodOptional<z.ZodEnum<{
@@ -10241,6 +10270,10 @@ declare const crawlerSchemas: {
10241
10270
  indicators: z.ZodArray<z.ZodString>;
10242
10271
  formActionPath: z.ZodOptional<z.ZodString>;
10243
10272
  }, z.core.$strip>>;
10273
+ metaRefresh: z.ZodOptional<z.ZodObject<{
10274
+ targetUrl: z.ZodString;
10275
+ delaySeconds: z.ZodNumber;
10276
+ }, z.core.$strip>>;
10244
10277
  tabbedContent: z.ZodOptional<z.ZodObject<{
10245
10278
  groups: z.ZodNumber;
10246
10279
  panels: z.ZodNumber;
@@ -11634,6 +11667,7 @@ declare const crawlerSchemas: {
11634
11667
  "robots-blocked": "robots-blocked";
11635
11668
  "canonical-conflict": "canonical-conflict";
11636
11669
  "canonical-hint-other": "canonical-hint-other";
11670
+ "meta-refresh": "meta-refresh";
11637
11671
  "not-html": "not-html";
11638
11672
  }>>;
11639
11673
  extractionStatus: z.ZodOptional<z.ZodEnum<{
@@ -11685,6 +11719,10 @@ declare const crawlerSchemas: {
11685
11719
  indicators: z.ZodArray<z.ZodString>;
11686
11720
  formActionPath: z.ZodOptional<z.ZodString>;
11687
11721
  }, z.core.$strip>>;
11722
+ metaRefresh: z.ZodOptional<z.ZodObject<{
11723
+ targetUrl: z.ZodString;
11724
+ delaySeconds: z.ZodNumber;
11725
+ }, z.core.$strip>>;
11688
11726
  tabbedContent: z.ZodOptional<z.ZodObject<{
11689
11727
  groups: z.ZodNumber;
11690
11728
  panels: z.ZodNumber;
@@ -13078,6 +13116,7 @@ declare const crawlerSchemas: {
13078
13116
  "robots-blocked": "robots-blocked";
13079
13117
  "canonical-conflict": "canonical-conflict";
13080
13118
  "canonical-hint-other": "canonical-hint-other";
13119
+ "meta-refresh": "meta-refresh";
13081
13120
  "not-html": "not-html";
13082
13121
  }>>;
13083
13122
  extractionStatus: z.ZodOptional<z.ZodEnum<{
@@ -13129,6 +13168,10 @@ declare const crawlerSchemas: {
13129
13168
  indicators: z.ZodArray<z.ZodString>;
13130
13169
  formActionPath: z.ZodOptional<z.ZodString>;
13131
13170
  }, z.core.$strip>>;
13171
+ metaRefresh: z.ZodOptional<z.ZodObject<{
13172
+ targetUrl: z.ZodString;
13173
+ delaySeconds: z.ZodNumber;
13174
+ }, z.core.$strip>>;
13132
13175
  tabbedContent: z.ZodOptional<z.ZodObject<{
13133
13176
  groups: z.ZodNumber;
13134
13177
  panels: z.ZodNumber;
@@ -14656,6 +14699,7 @@ declare const crawlerSchemas: {
14656
14699
  "robots-blocked": "robots-blocked";
14657
14700
  "canonical-conflict": "canonical-conflict";
14658
14701
  "canonical-hint-other": "canonical-hint-other";
14702
+ "meta-refresh": "meta-refresh";
14659
14703
  "not-html": "not-html";
14660
14704
  }>>;
14661
14705
  extractionStatus: z.ZodOptional<z.ZodEnum<{
@@ -14707,6 +14751,10 @@ declare const crawlerSchemas: {
14707
14751
  indicators: z.ZodArray<z.ZodString>;
14708
14752
  formActionPath: z.ZodOptional<z.ZodString>;
14709
14753
  }, z.core.$strip>>;
14754
+ metaRefresh: z.ZodOptional<z.ZodObject<{
14755
+ targetUrl: z.ZodString;
14756
+ delaySeconds: z.ZodNumber;
14757
+ }, z.core.$strip>>;
14710
14758
  tabbedContent: z.ZodOptional<z.ZodObject<{
14711
14759
  groups: z.ZodNumber;
14712
14760
  panels: z.ZodNumber;
@@ -15435,6 +15483,7 @@ declare const crawlerJsonSchemas: {
15435
15483
  "robots-blocked": "robots-blocked";
15436
15484
  "canonical-conflict": "canonical-conflict";
15437
15485
  "canonical-hint-other": "canonical-hint-other";
15486
+ "meta-refresh": "meta-refresh";
15438
15487
  "not-html": "not-html";
15439
15488
  }>>;
15440
15489
  extractionStatus: z.ZodOptional<z.ZodEnum<{
@@ -15486,6 +15535,10 @@ declare const crawlerJsonSchemas: {
15486
15535
  indicators: z.ZodArray<z.ZodString>;
15487
15536
  formActionPath: z.ZodOptional<z.ZodString>;
15488
15537
  }, z.core.$strip>>;
15538
+ metaRefresh: z.ZodOptional<z.ZodObject<{
15539
+ targetUrl: z.ZodString;
15540
+ delaySeconds: z.ZodNumber;
15541
+ }, z.core.$strip>>;
15489
15542
  tabbedContent: z.ZodOptional<z.ZodObject<{
15490
15543
  groups: z.ZodNumber;
15491
15544
  panels: z.ZodNumber;
@@ -16879,6 +16932,7 @@ declare const crawlerJsonSchemas: {
16879
16932
  "robots-blocked": "robots-blocked";
16880
16933
  "canonical-conflict": "canonical-conflict";
16881
16934
  "canonical-hint-other": "canonical-hint-other";
16935
+ "meta-refresh": "meta-refresh";
16882
16936
  "not-html": "not-html";
16883
16937
  }>>;
16884
16938
  extractionStatus: z.ZodOptional<z.ZodEnum<{
@@ -16930,6 +16984,10 @@ declare const crawlerJsonSchemas: {
16930
16984
  indicators: z.ZodArray<z.ZodString>;
16931
16985
  formActionPath: z.ZodOptional<z.ZodString>;
16932
16986
  }, z.core.$strip>>;
16987
+ metaRefresh: z.ZodOptional<z.ZodObject<{
16988
+ targetUrl: z.ZodString;
16989
+ delaySeconds: z.ZodNumber;
16990
+ }, z.core.$strip>>;
16933
16991
  tabbedContent: z.ZodOptional<z.ZodObject<{
16934
16992
  groups: z.ZodNumber;
16935
16993
  panels: z.ZodNumber;
@@ -18323,6 +18381,7 @@ declare const crawlerJsonSchemas: {
18323
18381
  "robots-blocked": "robots-blocked";
18324
18382
  "canonical-conflict": "canonical-conflict";
18325
18383
  "canonical-hint-other": "canonical-hint-other";
18384
+ "meta-refresh": "meta-refresh";
18326
18385
  "not-html": "not-html";
18327
18386
  }>>;
18328
18387
  extractionStatus: z.ZodOptional<z.ZodEnum<{
@@ -18374,6 +18433,10 @@ declare const crawlerJsonSchemas: {
18374
18433
  indicators: z.ZodArray<z.ZodString>;
18375
18434
  formActionPath: z.ZodOptional<z.ZodString>;
18376
18435
  }, z.core.$strip>>;
18436
+ metaRefresh: z.ZodOptional<z.ZodObject<{
18437
+ targetUrl: z.ZodString;
18438
+ delaySeconds: z.ZodNumber;
18439
+ }, z.core.$strip>>;
18377
18440
  tabbedContent: z.ZodOptional<z.ZodObject<{
18378
18441
  groups: z.ZodNumber;
18379
18442
  panels: z.ZodNumber;
@@ -19901,6 +19964,7 @@ declare const crawlerJsonSchemas: {
19901
19964
  "robots-blocked": "robots-blocked";
19902
19965
  "canonical-conflict": "canonical-conflict";
19903
19966
  "canonical-hint-other": "canonical-hint-other";
19967
+ "meta-refresh": "meta-refresh";
19904
19968
  "not-html": "not-html";
19905
19969
  }>>;
19906
19970
  extractionStatus: z.ZodOptional<z.ZodEnum<{
@@ -19952,6 +20016,10 @@ declare const crawlerJsonSchemas: {
19952
20016
  indicators: z.ZodArray<z.ZodString>;
19953
20017
  formActionPath: z.ZodOptional<z.ZodString>;
19954
20018
  }, z.core.$strip>>;
20019
+ metaRefresh: z.ZodOptional<z.ZodObject<{
20020
+ targetUrl: z.ZodString;
20021
+ delaySeconds: z.ZodNumber;
20022
+ }, z.core.$strip>>;
19955
20023
  tabbedContent: z.ZodOptional<z.ZodObject<{
19956
20024
  groups: z.ZodNumber;
19957
20025
  panels: z.ZodNumber;
package/dist/index.js CHANGED
@@ -1,5 +1,5 @@
1
1
  import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
2
- import { n as load, t as version } from "./package-B_swz2Xd.js";
2
+ import { n as load, t as version } from "./package-Cp8DwqCy.js";
3
3
  import { Headers, fetch as fetch$1 } from "undici";
4
4
  import { chmodSync, createReadStream, createWriteStream, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
5
5
  import { arch, homedir, platform, totalmem } from "node:os";
@@ -3176,6 +3176,22 @@ function extractStructuredData($, baseUrl, options = {}) {
3176
3176
  }
3177
3177
  //#endregion
3178
3178
  //#region packages/core/src/extract/page-extractor.ts
3179
+ function metaRefresh($, finalUrl) {
3180
+ for (const element of $("meta[http-equiv]").toArray()) {
3181
+ if ($(element).attr("http-equiv")?.trim().toLowerCase() !== "refresh") continue;
3182
+ const match = ($(element).attr("content")?.trim())?.match(/^([0-9]+(?:\.[0-9]+)?)\s*;\s*url\s*=\s*(?:"([^"]*)"|'([^']*)'|(.+))$/iu);
3183
+ if (!match) continue;
3184
+ try {
3185
+ const target = new URL((match[2] ?? match[3] ?? match[4] ?? "").trim(), finalUrl);
3186
+ if (!["http:", "https:"].includes(target.protocol)) continue;
3187
+ target.hash = "";
3188
+ return {
3189
+ targetUrl: target.toString(),
3190
+ delaySeconds: Number(match[1])
3191
+ };
3192
+ } catch {}
3193
+ }
3194
+ }
3179
3195
  const SOCIAL_PROFILE_HOSTS = [
3180
3196
  "facebook.com",
3181
3197
  "instagram.com",
@@ -3455,6 +3471,8 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
3455
3471
  }, extractor, dependencies);
3456
3472
  const { text, excerpt } = content;
3457
3473
  const title = safeText$1($("title").first().text());
3474
+ const crawlerMetaRefresh = extractor === "crawler" ? metaRefresh($, fetchResult.finalUrl) : void 0;
3475
+ if (crawlerMetaRefresh && new URL(crawlerMetaRefresh.targetUrl).origin === url.origin) crawlerInternalLinks.add(crawlerMetaRefresh.targetUrl);
3458
3476
  return {
3459
3477
  url: fetchResult.url,
3460
3478
  finalUrl: fetchResult.finalUrl,
@@ -3480,6 +3498,7 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
3480
3498
  externalAnchorSamples,
3481
3499
  socialProfileLinks: [...socialProfileLinks],
3482
3500
  contentSketch: contentSketch(text),
3501
+ metaRefresh: crawlerMetaRefresh,
3483
3502
  softAuthenticationGate: softAuthenticationGate($, fetchResult.finalUrl, title),
3484
3503
  tabbedContent: tabbedContent($)
3485
3504
  } } : {},
@@ -8289,6 +8308,9 @@ function failedMarkdownUrls(discovery) {
8289
8308
  return !negotiatedOk || !explicitOk || page.repeatedHashStable !== true;
8290
8309
  }).map((page) => page.htmlUrl);
8291
8310
  }
8311
+ function unstableMarkdownUrls(discovery) {
8312
+ return discovery.markdownAlternates.pages.filter((page) => confirmedMarkdown(page) !== void 0 && page.repeatedHashStable !== true).map((page) => page.htmlUrl);
8313
+ }
8292
8314
  function qualityFailureUrls(discovery) {
8293
8315
  return discovery.markdownAlternates.pages.filter((page) => {
8294
8316
  const quality = page.quality;
@@ -8302,7 +8324,10 @@ function confirmedMarkdown(page) {
8302
8324
  function markdownChecks(discovery) {
8303
8325
  const markdown = discovery.markdownAlternates;
8304
8326
  const failedUrls = failedMarkdownUrls(discovery);
8327
+ const unstableUrls = unstableMarkdownUrls(discovery);
8305
8328
  const qualityUrls = qualityFailureUrls(discovery);
8329
+ const hasEligibleHtml = markdown.eligibleHtmlPages > 0;
8330
+ const hasEvaluatedMarkdown = markdown.evaluatedPages > 0;
8306
8331
  const negotiatedPages = markdown.pages.filter((page) => page.negotiated?.status === 200 && /^\s*text\/markdown\b/iu.test(page.negotiated.contentType ?? "")).length;
8307
8332
  const explicitPages = markdown.pages.filter((page) => page.htmlAlternateUnique).length;
8308
8333
  const coverageComplete = markdown.eligibleHtmlPages > 0 && markdown.evaluatedPages === markdown.eligibleHtmlPages;
@@ -8331,25 +8356,28 @@ function markdownChecks(discovery) {
8331
8356
  check$2("representations", {
8332
8357
  id: "markdown-coverage",
8333
8358
  status: markdown.eligibleHtmlPages === 0 ? "unknown" : coverageComplete ? "pass" : "fail",
8334
- title: coverageComplete ? "Every successful HTML page has a Markdown representation" : "Markdown representation coverage is incomplete",
8335
- plainEnglish: `${markdown.evaluatedPages} of ${markdown.eligibleHtmlPages} successful HTML pages returned Markdown. ${explicitPages} advertised an explicit alternative and ${negotiatedPages} supported content negotiation.`,
8336
- action: coverageComplete ? "Keep the negotiated or explicit Markdown response stable as routes change." : "Return text/markdown through content negotiation or advertise one working Markdown alternative for each public HTML page.",
8359
+ title: coverageComplete ? "Every successful HTML page has a Markdown representation" : !hasEligibleHtml ? "Markdown representation coverage was not evaluated" : "Markdown representation coverage is incomplete",
8360
+ plainEnglish: hasEligibleHtml ? `${markdown.evaluatedPages} of ${markdown.eligibleHtmlPages} successful HTML pages returned Markdown. ${explicitPages} advertised an explicit alternative and ${negotiatedPages} supported content negotiation.` : "No successful HTML page was available, so Markdown representation coverage could not be evaluated.",
8361
+ action: !hasEligibleHtml ? "Resolve the crawl or page-fetch failure, then run this check again." : coverageComplete ? "Keep the negotiated or explicit Markdown response stable as routes change." : "Return text/markdown through content negotiation or advertise one working Markdown alternative for each public HTML page.",
8337
8362
  evidence: {
8338
8363
  eligibleHtmlPages: markdown.eligibleHtmlPages,
8339
8364
  advertisedPages: markdown.advertisedPages,
8340
8365
  negotiatedPages,
8341
8366
  evaluatedPages: markdown.evaluatedPages
8342
8367
  },
8343
- urls: failedUrls.slice(0, 25)
8368
+ urls: hasEligibleHtml ? failedUrls.slice(0, 25) : []
8344
8369
  }),
8345
8370
  check$2("representations", {
8346
8371
  id: "markdown-token-estimates",
8347
8372
  status: markdown.evaluatedPages === 0 ? "unknown" : tokenHeaders === markdown.evaluatedPages ? "pass" : "info",
8348
- title: tokenHeaders === markdown.evaluatedPages ? "Every Markdown response includes a token estimate" : "Some Markdown responses are missing a stable token estimate",
8349
- plainEnglish: `${tokenHeaders} of ${markdown.evaluatedPages} evaluated pages returned X-Markdown-Tokens. Paired explicit and negotiated responses agreed where both existed. The value is an estimate, not an exact model-specific token count.`,
8350
- action: tokenHeaders === markdown.evaluatedPages ? "Keep the estimate stable when both explicit and negotiated responses exist." : "No change is required. X-Markdown-Tokens is optional; local size estimates are reported separately.",
8373
+ title: !hasEvaluatedMarkdown ? "Markdown token estimates were not evaluated" : tokenHeaders === markdown.evaluatedPages ? "Every Markdown response includes a token estimate" : "Some Markdown responses are missing a stable token estimate",
8374
+ plainEnglish: hasEvaluatedMarkdown ? `${tokenHeaders} of ${markdown.evaluatedPages} evaluated pages returned X-Markdown-Tokens. Paired explicit and negotiated responses agreed where both existed. The value is an estimate, not an exact model-specific token count.` : "No confirmed Markdown response was available, so optional server token estimates could not be evaluated.",
8375
+ action: !hasEvaluatedMarkdown ? "Confirm a successful text/markdown representation, then run this check again." : tokenHeaders === markdown.evaluatedPages ? "Keep the estimate stable when both explicit and negotiated responses exist." : "No change is required. X-Markdown-Tokens is optional; local size estimates are reported separately.",
8351
8376
  urls: markdown.pages.filter((page) => {
8352
- return confirmedMarkdown(page)?.markdownTokens === void 0 || page.explicit !== void 0 && page.negotiated !== void 0 && page.explicit.markdownTokens !== page.negotiated.markdownTokens;
8377
+ const primary = confirmedMarkdown(page);
8378
+ if (!primary) return false;
8379
+ const paired = [page.explicit, page.negotiated].filter((response) => response?.status !== void 0 && response.status >= 200 && response.status < 300 && /^\s*text\/markdown\b/iu.test(response.contentType ?? ""));
8380
+ return primary?.markdownTokens === void 0 || paired.length === 2 && paired[0]?.markdownTokens !== paired[1]?.markdownTokens;
8353
8381
  }).map((page) => page.htmlUrl).slice(0, 25)
8354
8382
  }),
8355
8383
  check$2("representations", {
@@ -8357,7 +8385,7 @@ function markdownChecks(discovery) {
8357
8385
  status: sizedPages.length === 0 ? "unknown" : largePages.length === 0 ? "pass" : "warning",
8358
8386
  title: sizedPages.length === 0 ? "Markdown size was not measured" : largePages.length === 0 ? "Markdown representations stay within the review threshold" : "Some Markdown representations are large enough to review",
8359
8387
  plainEnglish: `${sizedPages.length} Markdown responses were measured locally. ${largePages.length} exceeded 50,000 characters and ${veryLargePages.length} exceeded 100,000 characters. These are processing-risk thresholds, not universal model limits.`,
8360
- action: largePages.length === 0 ? "Keep important content early and retain stable section structure as pages grow." : "Split genuinely separate topics, keep navigation and repeated chrome out of Markdown, and put the most useful content early. Do not remove necessary detail just to hit a threshold.",
8388
+ action: sizedPages.length === 0 ? "Confirm a successful text/markdown representation, then run this check again." : largePages.length === 0 ? "Keep important content early and retain stable section structure as pages grow." : "Split genuinely separate topics, keep navigation and repeated chrome out of Markdown, and put the most useful content early. Do not remove necessary detail just to hit a threshold.",
8361
8389
  evidence: {
8362
8390
  measuredPages: sizedPages.length,
8363
8391
  reviewThresholdCharacters: 5e4,
@@ -8375,7 +8403,7 @@ function markdownChecks(discovery) {
8375
8403
  status: parityPages.length === 0 ? "unknown" : parityFailures.length === 0 ? "pass" : "warning",
8376
8404
  title: parityPages.length === 0 ? "HTML and Markdown content parity was not measured" : parityFailures.length === 0 ? "Markdown retains sampled content from the HTML document" : "Some Markdown responses may omit important HTML content",
8377
8405
  plainEnglish: `${parityPages.length} pages were compared using bounded hashes sampled across the main HTML content. ${parityFailures.length} retained less than 60% of that sample. This detects likely omissions without storing full page bodies.`,
8378
- action: parityFailures.length === 0 ? "Keep the shared conversion path covered as templates and components change." : "Compare the affected HTML and Markdown around the missing sections, then fix the shared converter or component serialization.",
8406
+ action: parityPages.length === 0 ? "Confirm a successful text/markdown representation, then run this check again." : parityFailures.length === 0 ? "Keep the shared conversion path covered as templates and components change." : "Compare the affected HTML and Markdown around the missing sections, then fix the shared converter or component serialization.",
8379
8407
  evidence: {
8380
8408
  evaluatedPages: parityPages.length,
8381
8409
  failedPages: parityFailures.length,
@@ -8400,32 +8428,32 @@ function markdownChecks(discovery) {
8400
8428
  check$2("representations", {
8401
8429
  id: "markdown-negotiation",
8402
8430
  status: markdown.eligibleHtmlPages === 0 ? "unknown" : negotiationComplete && discovery.contentNegotiation.qZeroHonoured ? "pass" : "fail",
8403
- title: negotiationComplete && discovery.contentNegotiation.qZeroHonoured ? "Markdown content negotiation works across the site" : "Markdown content negotiation needs attention",
8404
- plainEnglish: `${negotiatedPages} of ${markdown.eligibleHtmlPages} pages returned negotiated Markdown. ${markdown.exactByteMatches} of ${explicitPages} paired explicit responses matched byte for byte. A request that refuses Markdown ${discovery.contentNegotiation.qZeroHonoured ? "received HTML" : "did not produce confirmed HTML evidence"}.`,
8405
- action: "Honour Accept q-values, send Vary: Accept, and keep paired explicit and negotiated responses identical when both exist.",
8431
+ title: !hasEligibleHtml ? "Markdown content negotiation was not evaluated" : negotiationComplete && discovery.contentNegotiation.qZeroHonoured ? "Markdown content negotiation works across the site" : "Markdown content negotiation needs attention",
8432
+ plainEnglish: hasEligibleHtml ? `${negotiatedPages} of ${markdown.eligibleHtmlPages} pages returned negotiated Markdown. ${markdown.exactByteMatches} of ${explicitPages} paired explicit responses matched byte for byte. A request that refuses Markdown ${discovery.contentNegotiation.qZeroHonoured ? "received HTML" : "did not produce confirmed HTML evidence"}.` : "No successful HTML page was available, so content negotiation could not be evaluated.",
8433
+ action: hasEligibleHtml ? "Honour Accept q-values, send Vary: Accept, and keep paired explicit and negotiated responses identical when both exist." : "Resolve the crawl or page-fetch failure, then run this check again.",
8406
8434
  evidence: {
8407
8435
  negotiatedPages,
8408
8436
  explicitPages,
8409
8437
  exactByteMatches: markdown.exactByteMatches,
8410
8438
  qZero: discovery.contentNegotiation
8411
8439
  },
8412
- urls: failedUrls.slice(0, 25)
8440
+ urls: hasEligibleHtml ? failedUrls.slice(0, 25) : []
8413
8441
  }),
8414
8442
  check$2("representations", {
8415
8443
  id: "markdown-determinism",
8416
8444
  status: markdown.evaluatedPages === 0 ? "unknown" : stable ? "pass" : "fail",
8417
- title: stable ? "Repeated Markdown responses are stable" : "Repeated Markdown responses changed during the audit",
8418
- plainEnglish: `${markdown.stableResponses} of ${markdown.evaluatedPages} repeated Markdown responses kept the same SHA-256 digest.`,
8419
- action: "Remove timestamps, random output, or other runtime rewriting that changes the same Markdown response between requests.",
8445
+ title: !hasEvaluatedMarkdown ? "Markdown response stability was not evaluated" : stable ? "Repeated Markdown responses are stable" : "Repeated Markdown responses changed during the audit",
8446
+ plainEnglish: hasEvaluatedMarkdown ? `${markdown.stableResponses} of ${markdown.evaluatedPages} repeated Markdown responses kept the same SHA-256 digest.` : "No confirmed Markdown response was available for a repeated stability check.",
8447
+ action: !hasEvaluatedMarkdown ? "Confirm a successful text/markdown representation, then run this check again." : stable ? "Keep repeated Markdown responses byte-stable as templates and data change." : "Remove timestamps, random output, or other runtime rewriting that changes the same Markdown response between requests.",
8420
8448
  evidence: { stableResponses: markdown.stableResponses },
8421
- urls: failedUrls.slice(0, 25)
8449
+ urls: unstableUrls.slice(0, 25)
8422
8450
  }),
8423
8451
  check$2("representations", {
8424
8452
  id: "markdown-quality",
8425
8453
  status: markdown.evaluatedPages === 0 ? "unknown" : qualityUrls.length === 0 ? "pass" : "warning",
8426
- title: qualityUrls.length === 0 ? "Markdown keeps the useful document structure cleanly" : "Some Markdown alternatives need an extraction review",
8427
- plainEnglish: qualityUrls.length === 0 ? "The evaluated alternatives retained one H1, frontmatter titles, balanced code fences, useful copy, and no leaked SVG, script, style, or layout tags." : `${qualityUrls.length} Markdown alternative${qualityUrls.length === 1 ? "" : "s"} lost important structure, looked navigation-only, or leaked presentation markup.`,
8428
- action: "Open the affected Markdown directly and fix the shared converter or component exclusion rules rather than patching generated files.",
8454
+ title: !hasEvaluatedMarkdown ? "Markdown quality was not evaluated" : qualityUrls.length === 0 ? "Markdown keeps the useful document structure cleanly" : "Some Markdown alternatives need an extraction review",
8455
+ plainEnglish: !hasEvaluatedMarkdown ? "No confirmed Markdown response was available for structure and extraction checks." : qualityUrls.length === 0 ? "The evaluated alternatives retained one H1, frontmatter titles, balanced code fences, useful copy, and no leaked SVG, script, style, or layout tags." : `${qualityUrls.length} Markdown alternative${qualityUrls.length === 1 ? "" : "s"} lost important structure, looked navigation-only, or leaked presentation markup.`,
8456
+ action: !hasEvaluatedMarkdown ? "Confirm a successful text/markdown representation, then run this check again." : qualityUrls.length === 0 ? "Keep the shared Markdown conversion path covered as templates and components change." : "Open the affected Markdown directly and fix the shared converter or component exclusion rules rather than patching generated files.",
8429
8457
  urls: qualityUrls.slice(0, 25)
8430
8458
  })
8431
8459
  ];
@@ -10349,7 +10377,8 @@ const RULE_DEFINITIONS = [
10349
10377
  evidenceFields: [
10350
10378
  "page.url",
10351
10379
  "page.finalUrl",
10352
- "page.status"
10380
+ "page.status",
10381
+ "page.metaRefresh"
10353
10382
  ],
10354
10383
  suggestedCommands: ["seo redirect-trace --url <url> --json"]
10355
10384
  }
@@ -10846,6 +10875,7 @@ function comparableUrl(value) {
10846
10875
  }
10847
10876
  }
10848
10877
  function isRedirectedPage(page) {
10878
+ if (page.metaRefresh) return true;
10849
10879
  const requested = comparableUrl(page.url);
10850
10880
  const final = comparableUrl(page.finalUrl);
10851
10881
  return Boolean(requested && final && requested !== final);
@@ -11120,11 +11150,18 @@ function auditCrawlPages(pages, opts = {}) {
11120
11150
  }
11121
11151
  if (page.status < 200) continue;
11122
11152
  const redirected = isRedirectedPage(page);
11123
- if (redirected) issues.push(issue("redirected_url", page, `Final URL: ${page.finalUrl}`, {
11124
- requestedUrl: page.url,
11125
- finalUrl: page.finalUrl,
11126
- status: page.status
11127
- }));
11153
+ if (redirected) {
11154
+ const redirectTarget = page.metaRefresh?.targetUrl ?? page.finalUrl;
11155
+ issues.push(issue("redirected_url", page, `Final URL: ${redirectTarget}`, {
11156
+ requestedUrl: page.url,
11157
+ finalUrl: redirectTarget,
11158
+ status: page.status,
11159
+ ...page.metaRefresh ? {
11160
+ redirectType: "meta-refresh",
11161
+ delaySeconds: page.metaRefresh.delaySeconds
11162
+ } : {}
11163
+ }));
11164
+ }
11128
11165
  const redirectChain = page.fetchDiagnostics?.redirectChain ?? [];
11129
11166
  if (redirectChain.length > 1) issues.push(issue("redirect_chain", page, `${redirectChain.length} hops`, {
11130
11167
  redirectChain,
@@ -11980,7 +12017,7 @@ function latestCrawlReport(site, store = crawlReportStore) {
11980
12017
  //#region packages/core/src/analyze/crawler/crawl-caveats.ts
11981
12018
  function crawlCaveats(input) {
11982
12019
  const caveats = input.cancelled ? ["Crawl cancelled before all queued URLs finished."] : input.pageLimitReached ? [`Stopped after reaching maxPages (${input.maxPages}).`] : [];
11983
- if (input.queueSafetySkippedUrls > 0) caveats.push(`Left ${input.queueSafetySkippedUrls} eligible same-origin URLs unqueued to keep this crawl bounded. Increase --max-pages to inspect more.`);
12020
+ if (input.queueSafetySkippedUrls > 0) caveats.push(input.queueSafetyInventoryCapped ? `Left at least ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. The excluded-URL inventory reached its reporting cap. Increase --max-pages to inspect more.` : `Left ${input.queueSafetySkippedUrls} unique eligible same-origin URLs unqueued to keep this crawl bounded. Increase --max-pages to inspect more.`);
11984
12021
  if (input.originBackpressureSkippedUrls > 0) caveats.push(`Stopped ${input.originBackpressureSkippedUrls} queued URL${input.originBackpressureSkippedUrls === 1 ? "" : "s"} because the origin stayed slow. This is incomplete crawl evidence, not a failed URL check.`);
11985
12022
  if (input.memoryPressureSkippedUrls > 0) caveats.push(`Left ${input.memoryPressureSkippedUrls} eligible URL${input.memoryPressureSkippedUrls === 1 ? "" : "s"} unchecked after the local memory safety limit was reached. Start with --health or lower --max-pages.`);
11986
12023
  return caveats;
@@ -13133,6 +13170,7 @@ async function crawlOne(url, opts = {}) {
13133
13170
  contentSample: truncate$1(extracted.contentText, 300),
13134
13171
  contentSketch: crawlerEvidence.contentSketch,
13135
13172
  softAuthenticationGate: crawlerEvidence.softAuthenticationGate,
13173
+ metaRefresh: crawlerEvidence.metaRefresh,
13136
13174
  tabbedContent: crawlerEvidence.tabbedContent,
13137
13175
  lang: extracted.lang,
13138
13176
  hasViewport: extracted.hasViewport,
@@ -13183,9 +13221,9 @@ async function crawlOne(url, opts = {}) {
13183
13221
  hasQapageSchema: extracted.schemaTypes.includes("QAPage")
13184
13222
  }
13185
13223
  };
13186
- page.indexability = indexabilityReason(page);
13224
+ page.indexability = page.metaRefresh ? `Meta refresh to ${page.metaRefresh.targetUrl}` : indexabilityReason(page);
13187
13225
  page.indexable = !page.indexability;
13188
- page.declaredIndexability = hasMetaRobotsDirective(page.metaRobots, "noindex") || hasXRobotsDirective(page.xRobotsTag, "noindex") ? "noindex" : page.robotsTxt?.allowed === false ? "robots-blocked" : page.canonicalStatus === "conflicting" ? "canonical-conflict" : page.canonical && !sameDocumentUrl(page.canonical, page.finalUrl) ? "canonical-hint-other" : "indexable-candidate";
13226
+ page.declaredIndexability = page.metaRefresh ? "meta-refresh" : hasMetaRobotsDirective(page.metaRobots, "noindex") || hasXRobotsDirective(page.xRobotsTag, "noindex") ? "noindex" : page.robotsTxt?.allowed === false ? "robots-blocked" : page.canonicalStatus === "conflicting" ? "canonical-conflict" : page.canonical && !sameDocumentUrl(page.canonical, page.finalUrl) ? "canonical-hint-other" : "indexable-candidate";
13189
13227
  return {
13190
13228
  request: responseObservation(url, fetched, "complete"),
13191
13229
  page,
@@ -13979,6 +14017,9 @@ async function crawlSite(input, dependencies) {
13979
14017
  let robotsDeferredUrls = 0;
13980
14018
  let originBackpressureSkippedUrls = 0;
13981
14019
  let queueSafetySkippedUrls = 0;
14020
+ const queueSafetyDeferredUrls = /* @__PURE__ */ new Set();
14021
+ const maxTrackedQueueSafetyUrls = Math.min(5e4, Math.max(1e3, config.maxPages * 5));
14022
+ let queueSafetyInventoryCapped = false;
13982
14023
  let failedUrls = 0;
13983
14024
  let observedInternalLinks = 0;
13984
14025
  let memoryLimited = false;
@@ -14083,13 +14124,16 @@ async function crawlSite(input, dependencies) {
14083
14124
  return;
14084
14125
  }
14085
14126
  if (queue.size + visited.size + inFlight.size >= config.maxPages * 5) {
14086
- recordSkip("queue-safety-limit");
14087
- queueSafetySkippedUrls += 1;
14088
- emitStatus("url_skipped", {
14089
- url: normalized,
14090
- depth,
14091
- reason: "queue_safety_limit"
14092
- });
14127
+ if (!queueSafetyDeferredUrls.has(normalized)) if (queueSafetyDeferredUrls.size < maxTrackedQueueSafetyUrls) {
14128
+ queueSafetyDeferredUrls.add(normalized);
14129
+ queueSafetySkippedUrls = queueSafetyDeferredUrls.size;
14130
+ recordSkip("queue-safety-limit");
14131
+ emitStatus("url_skipped", {
14132
+ url: normalized,
14133
+ depth,
14134
+ reason: "queue_safety_limit"
14135
+ });
14136
+ } else queueSafetyInventoryCapped = true;
14093
14137
  return;
14094
14138
  }
14095
14139
  discovered.add(normalized);
@@ -14462,6 +14506,7 @@ async function crawlSite(input, dependencies) {
14462
14506
  pageLimitReached,
14463
14507
  maxPages: config.maxPages,
14464
14508
  queueSafetySkippedUrls,
14509
+ queueSafetyInventoryCapped,
14465
14510
  originBackpressureSkippedUrls,
14466
14511
  memoryPressureSkippedUrls
14467
14512
  }),
@@ -17356,6 +17401,7 @@ const crawlPageSnapshotSchema = z.object({
17356
17401
  "robots-blocked",
17357
17402
  "canonical-conflict",
17358
17403
  "canonical-hint-other",
17404
+ "meta-refresh",
17359
17405
  "not-html",
17360
17406
  "unknown"
17361
17407
  ]).optional(),
@@ -17402,6 +17448,10 @@ const crawlPageSnapshotSchema = z.object({
17402
17448
  indicators: z.array(z.string()),
17403
17449
  formActionPath: z.string().optional()
17404
17450
  }).optional(),
17451
+ metaRefresh: z.object({
17452
+ targetUrl: z.string().url(),
17453
+ delaySeconds: z.number().nonnegative()
17454
+ }).optional(),
17405
17455
  tabbedContent: z.object({
17406
17456
  groups: z.number().int().positive(),
17407
17457
  panels: z.number().int().positive(),