seo 0.2.1 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,5 +1,5 @@
1
1
  import { createRequire as __seoCreateRequire } from 'node:module'; const require = __seoCreateRequire(import.meta.url);
2
- import { n as load, t as version } from "./package-BTOI9tde.js";
2
+ import { n as load, t as version } from "./package-Jl7x3QBx.js";
3
3
  import { Headers, fetch as fetch$1 } from "undici";
4
4
  import { chmodSync, createWriteStream, existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
5
5
  import { homedir } from "node:os";
@@ -857,10 +857,8 @@ async function createGoogleAccessTokenClient() {
857
857
  }
858
858
  async function authStatus() {
859
859
  const status = getAuthModeStatus();
860
- const tokens = await readTokens();
861
860
  const serviceAccount = getServiceAccountStatus();
862
861
  if (serviceAccount.configured || serviceAccount.error) return {
863
- tokens,
864
862
  configured: serviceAccount.configured,
865
863
  sharedConfigured: status.sharedConfigured,
866
864
  byoConfigured: status.byoConfigured,
@@ -868,6 +866,7 @@ async function authStatus() {
868
866
  identity: serviceAccount.identity,
869
867
  serviceAccount
870
868
  };
869
+ const tokens = await readTokens();
871
870
  return {
872
871
  tokens,
873
872
  configured: Boolean(getClientConfig(tokens?.client_source)),
@@ -2219,7 +2218,7 @@ function effectiveSnippetControl(input) {
2219
2218
  }
2220
2219
  //#endregion
2221
2220
  //#region packages/core/src/extract/canonical.ts
2222
- function headerValue$3(headers, name) {
2221
+ function headerValue$4(headers, name) {
2223
2222
  const lower = name.toLowerCase();
2224
2223
  return Object.entries(headers).find(([key]) => key.toLowerCase() === lower)?.[1];
2225
2224
  }
@@ -2303,7 +2302,7 @@ function parameter(value, name) {
2303
2302
  return match?.[1] ?? match?.[2] ?? match?.[3];
2304
2303
  }
2305
2304
  function headerCandidates(headers, baseUrl) {
2306
- const link = headerValue$3(headers, "link");
2305
+ const link = headerValue$4(headers, "link");
2307
2306
  if (!link) return [];
2308
2307
  const candidates = [];
2309
2308
  for (const item of splitLinkHeader(link)) {
@@ -2972,7 +2971,7 @@ function safeText$1(value) {
2972
2971
  const trimmed = value?.replace(/\s+/g, " ").trim();
2973
2972
  return trimmed ? trimmed : void 0;
2974
2973
  }
2975
- function headerValue$2(headers, name) {
2974
+ function headerValue$3(headers, name) {
2976
2975
  const target = name.toLowerCase();
2977
2976
  return Object.entries(headers).find(([key]) => key.toLowerCase() === target)?.[1];
2978
2977
  }
@@ -3071,6 +3070,7 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
3071
3070
  });
3072
3071
  else invalidHreflangCount += 1;
3073
3072
  }
3073
+ const markdownAlternates = [...new Set($("head link[rel~=\"alternate\"][type][href]").toArray().filter((element) => $(element).attr("type")?.trim().toLowerCase() === "text/markdown").map((element) => httpUrl($(element).attr("href"), fetchResult.finalUrl)).filter((value) => Boolean(value)))].sort();
3074
3074
  const openGraph = Object.fromEntries($("meta[property^=\"og:\"]").toArray().map((element) => [$(element).attr("property") ?? "", $(element).attr("content") ?? ""]).filter(([key, value]) => key && value));
3075
3075
  const twitter = Object.fromEntries($("meta[name^=\"twitter:\"]").toArray().map((element) => [$(element).attr("name") ?? "", $(element).attr("content") ?? ""]).filter(([key, value]) => key && value));
3076
3076
  const structuredData = extractStructuredData($, fetchResult.finalUrl);
@@ -3118,13 +3118,14 @@ async function extractPage(fetchResult, extractor = "defuddle", dependencies = {
3118
3118
  title: safeText$1($("title").first().text()),
3119
3119
  metaDescription: safeText$1($("meta[name=\"description\"]").attr("content")),
3120
3120
  metaRobots,
3121
- xRobotsTag: safeText$1(headerValue$2(fetchResult.headers, "x-robots-tag")),
3121
+ xRobotsTag: safeText$1(headerValue$3(fetchResult.headers, "x-robots-tag")),
3122
3122
  canonical: canonicalEvidence.selectedRaw,
3123
3123
  canonicalEvidence,
3124
3124
  lang: safeText$1($("html").attr("lang")),
3125
3125
  hasViewport: Boolean($("meta[name=\"viewport\"]").attr("content")),
3126
3126
  headings,
3127
3127
  links,
3128
+ markdownAlternates,
3128
3129
  hreflang,
3129
3130
  jsonLd: structuredData.jsonLd,
3130
3131
  invalidJsonLdCount: structuredData.invalidJsonLdSamples.length,
@@ -3929,7 +3930,7 @@ function safeText(value) {
3929
3930
  const normalized = value?.replace(/\s+/g, " ").trim();
3930
3931
  return normalized ? normalized.slice(0, MAX_TEXT_VALUE_LENGTH) : void 0;
3931
3932
  }
3932
- function headerValue$1(headers, name) {
3933
+ function headerValue$2(headers, name) {
3933
3934
  const expected = name.toLowerCase();
3934
3935
  return Object.entries(headers).find(([key]) => key.toLowerCase() === expected)?.[1];
3935
3936
  }
@@ -3982,7 +3983,7 @@ function renderingDocumentSnapshot(result) {
3982
3983
  const structuredData = extractStructuredData($, result.finalUrl);
3983
3984
  const metaRobots = normalizedDirectiveValue([metaValue($, "robots")]);
3984
3985
  const googlebotRobots = normalizedDirectiveValue([metaValue($, "googlebot")]);
3985
- const httpRobots = normalizedDirectiveValue([headerValue$1(result.headers, "x-robots-tag")]);
3986
+ const httpRobots = normalizedDirectiveValue([headerValue$2(result.headers, "x-robots-tag")]);
3986
3987
  const headings = $("h1, h2, h3, h4, h5, h6").toArray().map((element) => ({
3987
3988
  level: Number.parseInt(element.tagName.slice(1), 10),
3988
3989
  text: safeText($(element).text())
@@ -6627,24 +6628,1104 @@ function selectAffectedUrls(report, filters = {}) {
6627
6628
  const limit = filters.limit ?? 100;
6628
6629
  const selected = matches.slice(0, limit);
6629
6630
  return {
6630
- affectedUrls: selected,
6631
- selection: {
6632
- totalMatchedRows: matches.length,
6633
- returnedRows: selected.length,
6634
- limit,
6635
- truncated: selected.length < matches.length
6631
+ affectedUrls: selected,
6632
+ selection: {
6633
+ totalMatchedRows: matches.length,
6634
+ returnedRows: selected.length,
6635
+ limit,
6636
+ truncated: selected.length < matches.length
6637
+ }
6638
+ };
6639
+ }
6640
+ function matchesFilters(issue, filters) {
6641
+ if (filters.ruleId && issue.ruleId !== filters.ruleId) return false;
6642
+ if (filters.category && issue.category !== filters.category) return false;
6643
+ if (filters.severity && issue.severity !== filters.severity) return false;
6644
+ return true;
6645
+ }
6646
+ //#endregion
6647
+ //#region packages/core/src/analyze/crawler/agent-discovery.ts
6648
+ const MAX_BODY_BYTES = 2e6;
6649
+ const MAX_LLMS_LINKS = 100;
6650
+ const MAX_CURATED_LLMS_BYTES = 1e5;
6651
+ const MAX_SKILLS = 25;
6652
+ function sha256(value) {
6653
+ return createHash("sha256").update(value).digest("hex");
6654
+ }
6655
+ function headerValue$1(headers, name) {
6656
+ if (!headers) return void 0;
6657
+ const target = name.toLowerCase();
6658
+ return Object.entries(headers).find(([key]) => key.toLowerCase() === target)?.[1];
6659
+ }
6660
+ function linkEntries(value, base) {
6661
+ if (!value) return [];
6662
+ return value.split(/,(?=\s*<)/u).map((entry) => {
6663
+ const match = entry.match(/^\s*<([^>]+)>\s*(.*)$/u);
6664
+ if (!match?.[1]) return void 0;
6665
+ try {
6666
+ const url = new URL(match[1], base).toString();
6667
+ const parameters = match[2] ?? "";
6668
+ const rel = parameters.match(/(?:^|;)\s*rel=(?:"([^"]+)"|([^;\s]+))/iu);
6669
+ const type = parameters.match(/(?:^|;)\s*type=(?:"([^"]+)"|([^;\s]+))/iu);
6670
+ return {
6671
+ url,
6672
+ rel: (rel?.[1] ?? rel?.[2] ?? "").toLowerCase().split(/\s+/u),
6673
+ type: (type?.[1] ?? type?.[2] ?? "").toLowerCase()
6674
+ };
6675
+ } catch {
6676
+ return;
6677
+ }
6678
+ }).filter((entry) => Boolean(entry));
6679
+ }
6680
+ function normalizedDocumentUrl(value) {
6681
+ const url = new URL(value);
6682
+ url.hash = "";
6683
+ if (url.pathname !== "/") url.pathname = url.pathname.replace(/\/$/u, "");
6684
+ return url.toString();
6685
+ }
6686
+ function sameDocument(left, right) {
6687
+ if (!left || !right) return null;
6688
+ try {
6689
+ return normalizedDocumentUrl(left) === normalizedDocumentUrl(right);
6690
+ } catch {
6691
+ return false;
6692
+ }
6693
+ }
6694
+ function bodyWordCount(value) {
6695
+ return value.replace(/^---[\s\S]*?---\s*/u, "").replace(/```[\s\S]*?```/gu, " ").replace(/[#>*_`[\]()|:-]/gu, " ").split(/\s+/u).filter(Boolean).length;
6696
+ }
6697
+ function repeatedProseLines(markdown) {
6698
+ const counts = /* @__PURE__ */ new Map();
6699
+ let inFrontmatter = false;
6700
+ let fence;
6701
+ for (const [index, rawLine] of markdown.split(/\r?\n/u).entries()) {
6702
+ const line = rawLine.trim();
6703
+ if (index === 0 && line === "---") {
6704
+ inFrontmatter = true;
6705
+ continue;
6706
+ }
6707
+ if (inFrontmatter) {
6708
+ if (line === "---") inFrontmatter = false;
6709
+ continue;
6710
+ }
6711
+ if (!fence && (line.startsWith("```") || line.startsWith("~~~"))) {
6712
+ fence = line.startsWith("```") ? "```" : "~~~";
6713
+ continue;
6714
+ }
6715
+ if (fence) {
6716
+ if (line.startsWith(fence)) fence = void 0;
6717
+ continue;
6718
+ }
6719
+ if (line.length < 40 || /^(?:#{1,6}\s|[-+*]\s|\d+[.)]\s|>|\||`|\]\()/u.test(line)) continue;
6720
+ counts.set(line, (counts.get(line) ?? 0) + 1);
6721
+ }
6722
+ return [...counts.values()].reduce((duplicates, count) => duplicates + Math.max(0, count - 1), 0);
6723
+ }
6724
+ function responseIsNoindex(response, body) {
6725
+ if ((response.headers.get("x-robots-tag") ?? "").toLowerCase().includes("noindex")) return true;
6726
+ return [...body.matchAll(/<meta\s+[^>]*>/giu)].some((match) => {
6727
+ const tag = match[0];
6728
+ const name = tag.match(/\bname\s*=\s*["']([^"']+)["']/iu)?.[1];
6729
+ const content = tag.match(/\bcontent\s*=\s*["']([^"']+)["']/iu)?.[1];
6730
+ return name?.toLowerCase() === "robots" && /\bnoindex\b/iu.test(content ?? "");
6731
+ });
6732
+ }
6733
+ function markdownQuality(markdown, page) {
6734
+ const introProbe = (page.contentSample?.replace(/\s+/gu, " ").trim())?.split(" ").slice(0, 10).join(" ");
6735
+ const normalizedMarkdown = markdown.replace(/\s+/gu, " ");
6736
+ const codeFences = markdown.match(/^```/gmu)?.length ?? 0;
6737
+ const wordCount = bodyWordCount(markdown);
6738
+ const sourceWordCount = page.wordCount ?? 0;
6739
+ return {
6740
+ frontmatterTitle: /^---[\s\S]*?^title:\s*.+$/mu.test(markdown),
6741
+ h1Count: markdown.match(/^#\s+\S.+$/gmu)?.length ?? 0,
6742
+ codeFenceBalanced: codeFences % 2 === 0,
6743
+ tableRows: markdown.match(/^\s*\|.*\|\s*$/gmu)?.length ?? 0,
6744
+ links: markdown.match(/\[[^\]]+\]\([^)]+\)/gu)?.length ?? 0,
6745
+ wordCount,
6746
+ rawHtmlTags: markdown.match(/<(?:div|span|section|article|nav|button)\b/giu)?.length ?? 0,
6747
+ rawSvgTags: markdown.match(/<svg\b/giu)?.length ?? 0,
6748
+ rawScriptTags: markdown.match(/<script\b/giu)?.length ?? 0,
6749
+ rawStyleTags: markdown.match(/<style\b/giu)?.length ?? 0,
6750
+ suspiciousConcatenations: markdown.match(/\b[a-z]{4,}[A-Z][A-Za-z]{3,}\b/gu)?.length ?? 0,
6751
+ repeatedLines: repeatedProseLines(markdown),
6752
+ sourceWordCount,
6753
+ wordRetentionRatio: sourceWordCount > 0 ? Math.round(wordCount / sourceWordCount * 1e3) / 1e3 : null,
6754
+ introductoryCopyRetained: introProbe ? normalizedMarkdown.includes(introProbe) : null,
6755
+ navigationOnly: bodyWordCount(markdown) < 25
6756
+ };
6757
+ }
6758
+ function combinedSignal(timeoutMs, signal) {
6759
+ const controller = new AbortController();
6760
+ const abort = () => controller.abort();
6761
+ const timer = setTimeout(abort, timeoutMs);
6762
+ if (signal?.aborted) controller.abort();
6763
+ else signal?.addEventListener("abort", abort, { once: true });
6764
+ return {
6765
+ signal: controller.signal,
6766
+ cleanup: () => {
6767
+ clearTimeout(timer);
6768
+ signal?.removeEventListener("abort", abort);
6769
+ }
6770
+ };
6771
+ }
6772
+ async function readBoundedText(response) {
6773
+ if (!response.body) return "";
6774
+ const reader = response.body.getReader();
6775
+ const chunks = [];
6776
+ let size = 0;
6777
+ try {
6778
+ while (true) {
6779
+ const result = await reader.read();
6780
+ if (result.done) break;
6781
+ size += result.value.byteLength;
6782
+ if (size > MAX_BODY_BYTES) throw new Error(`Response exceeds ${MAX_BODY_BYTES} bytes.`);
6783
+ chunks.push(result.value);
6784
+ }
6785
+ } finally {
6786
+ await reader.cancel().catch(() => void 0);
6787
+ }
6788
+ return Buffer.concat(chunks).toString("utf8");
6789
+ }
6790
+ async function fetchRepresentation(input) {
6791
+ const controller = combinedSignal(input.timeoutMs, input.signal);
6792
+ try {
6793
+ const response = await input.fetch(input.url, {
6794
+ profile: "bot",
6795
+ redirect: "follow",
6796
+ headers: { accept: input.accept },
6797
+ signal: controller.signal
6798
+ });
6799
+ const body = await readBoundedText(response);
6800
+ const link = linkEntries(response.headers.get("link") ?? void 0, input.url);
6801
+ const tokens = Number(response.headers.get("x-markdown-tokens"));
6802
+ return {
6803
+ observation: {
6804
+ requestedUrl: input.url,
6805
+ finalUrl: response.url || input.url,
6806
+ status: response.status,
6807
+ contentType: response.headers.get("content-type") ?? void 0,
6808
+ bytes: Buffer.byteLength(body),
6809
+ sha256: sha256(body),
6810
+ canonicalUrl: link.find((entry) => entry.rel.includes("canonical"))?.url,
6811
+ alternateUrl: link.find((entry) => entry.rel.includes("alternate") && entry.type === "text/markdown")?.url,
6812
+ varyAccept: (response.headers.get("vary") ?? "").split(",").some((value) => value.trim().toLowerCase() === "accept"),
6813
+ ...Number.isInteger(tokens) && tokens >= 0 ? { markdownTokens: tokens } : {},
6814
+ contentSignal: response.headers.get("content-signal") ?? void 0
6815
+ },
6816
+ body
6817
+ };
6818
+ } catch (error) {
6819
+ return { observation: {
6820
+ requestedUrl: input.url,
6821
+ varyAccept: false,
6822
+ error: error instanceof Error ? error.message : String(error)
6823
+ } };
6824
+ } finally {
6825
+ controller.cleanup();
6826
+ }
6827
+ }
6828
+ async function inspectMarkdownPage(input) {
6829
+ const advertisedUrls = [...new Set(input.page.markdownAlternates ?? [])].sort();
6830
+ const httpAlternateUrls = linkEntries(headerValue$1(input.page.responseHeaders, "link"), input.page.finalUrl).filter((entry) => entry.rel.includes("alternate") && entry.type === "text/markdown").map((entry) => entry.url).sort();
6831
+ const observation = {
6832
+ htmlUrl: input.page.finalUrl,
6833
+ advertisedUrls,
6834
+ htmlAlternateUnique: advertisedUrls.length === 1,
6835
+ httpAlternateUrls,
6836
+ explicitMatchesNegotiated: null,
6837
+ repeatedHashStable: null,
6838
+ markdownCanonicalMatchesHtml: null
6839
+ };
6840
+ if (advertisedUrls.length !== 1) return observation;
6841
+ const explicitUrl = advertisedUrls[0];
6842
+ const [explicit, negotiated, repeated] = await Promise.all([
6843
+ fetchRepresentation({
6844
+ url: explicitUrl,
6845
+ accept: "text/markdown",
6846
+ timeoutMs: input.timeoutMs,
6847
+ fetch: input.fetch,
6848
+ signal: input.signal
6849
+ }),
6850
+ fetchRepresentation({
6851
+ url: input.page.finalUrl,
6852
+ accept: "text/markdown",
6853
+ timeoutMs: input.timeoutMs,
6854
+ fetch: input.fetch,
6855
+ signal: input.signal
6856
+ }),
6857
+ fetchRepresentation({
6858
+ url: explicitUrl,
6859
+ accept: "text/markdown",
6860
+ timeoutMs: input.timeoutMs,
6861
+ fetch: input.fetch,
6862
+ signal: input.signal
6863
+ })
6864
+ ]);
6865
+ observation.explicit = explicit.observation;
6866
+ observation.negotiated = negotiated.observation;
6867
+ observation.repeated = repeated.observation;
6868
+ observation.explicitMatchesNegotiated = explicit.observation.sha256 && negotiated.observation.sha256 ? explicit.observation.sha256 === negotiated.observation.sha256 : null;
6869
+ observation.repeatedHashStable = explicit.observation.sha256 && repeated.observation.sha256 ? explicit.observation.sha256 === repeated.observation.sha256 : null;
6870
+ observation.markdownCanonicalMatchesHtml = sameDocument(explicit.observation.canonicalUrl, input.page.finalUrl);
6871
+ if (explicit.body) observation.quality = markdownQuality(explicit.body, input.page);
6872
+ return observation;
6873
+ }
6874
+ async function fetchText(input) {
6875
+ const controller = combinedSignal(input.timeoutMs, input.signal);
6876
+ try {
6877
+ const response = await input.fetch(input.url, {
6878
+ profile: "bot",
6879
+ redirect: input.redirect ?? "follow",
6880
+ headers: input.accept ? { accept: input.accept } : void 0,
6881
+ signal: controller.signal
6882
+ });
6883
+ return {
6884
+ response,
6885
+ body: await readBoundedText(response)
6886
+ };
6887
+ } finally {
6888
+ controller.cleanup();
6889
+ }
6890
+ }
6891
+ function safeError(error) {
6892
+ return error instanceof Error ? error.message : String(error);
6893
+ }
6894
+ async function inspectAgentSkills(input) {
6895
+ const indexUrl = new URL("/.well-known/agent-skills/index.json", input.origin).toString();
6896
+ try {
6897
+ const { response, body } = await fetchText({
6898
+ url: indexUrl,
6899
+ timeoutMs: input.timeoutMs,
6900
+ fetch: input.fetch,
6901
+ signal: input.signal,
6902
+ accept: "application/json"
6903
+ });
6904
+ const contentType = response.headers.get("content-type") ?? void 0;
6905
+ const cors = response.headers.get("access-control-allow-origin") ?? void 0;
6906
+ if (response.status < 200 || response.status >= 300) return {
6907
+ indexUrl,
6908
+ status: response.status,
6909
+ contentType,
6910
+ validIndex: false,
6911
+ skills: []
6912
+ };
6913
+ const parsed = JSON.parse(body);
6914
+ const entries = Array.isArray(parsed.skills) ? parsed.skills.slice(0, MAX_SKILLS) : [];
6915
+ const skills = await Promise.all(entries.map(async (entry) => {
6916
+ const name = typeof entry.name === "string" ? entry.name : void 0;
6917
+ const declaredDigest = typeof entry.digest === "string" ? entry.digest : void 0;
6918
+ const relativeUrl = typeof entry.url === "string" ? entry.url : void 0;
6919
+ if (!relativeUrl) return {
6920
+ name,
6921
+ declaredDigest,
6922
+ digestMatches: null,
6923
+ frontmatterValid: null,
6924
+ sameOrigin: false,
6925
+ error: "Skill URL is missing."
6926
+ };
6927
+ const url = new URL(relativeUrl, indexUrl).toString();
6928
+ const sameOrigin = new URL(url).origin === new URL(input.origin).origin;
6929
+ if (!sameOrigin) return {
6930
+ name,
6931
+ url,
6932
+ declaredDigest,
6933
+ digestMatches: null,
6934
+ frontmatterValid: null,
6935
+ sameOrigin,
6936
+ error: "Skill URL is off origin."
6937
+ };
6938
+ try {
6939
+ const skill = await fetchText({
6940
+ url,
6941
+ timeoutMs: input.timeoutMs,
6942
+ fetch: input.fetch,
6943
+ signal: input.signal,
6944
+ accept: "text/markdown,text/plain;q=0.9"
6945
+ });
6946
+ const observedDigest = `sha256:${sha256(skill.body)}`;
6947
+ const frontmatterValid = /^---\s*\n[\s\S]*?^name:\s*\S.+$[\s\S]*?^description:\s*\S.+$[\s\S]*?^---\s*$/mu.test(skill.body);
6948
+ return {
6949
+ name,
6950
+ url,
6951
+ status: skill.response.status,
6952
+ contentType: skill.response.headers.get("content-type") ?? void 0,
6953
+ cors: skill.response.headers.get("access-control-allow-origin") ?? void 0,
6954
+ declaredDigest,
6955
+ observedDigest,
6956
+ digestMatches: declaredDigest ? declaredDigest === observedDigest : null,
6957
+ frontmatterValid,
6958
+ sameOrigin
6959
+ };
6960
+ } catch (error) {
6961
+ return {
6962
+ name,
6963
+ url,
6964
+ declaredDigest,
6965
+ digestMatches: null,
6966
+ frontmatterValid: null,
6967
+ sameOrigin,
6968
+ error: safeError(error)
6969
+ };
6970
+ }
6971
+ }));
6972
+ return {
6973
+ indexUrl,
6974
+ status: response.status,
6975
+ contentType,
6976
+ cors,
6977
+ validIndex: typeof parsed.$schema === "string" && Array.isArray(parsed.skills),
6978
+ skills
6979
+ };
6980
+ } catch (error) {
6981
+ return {
6982
+ indexUrl,
6983
+ validIndex: false,
6984
+ skills: [],
6985
+ error: safeError(error)
6986
+ };
6987
+ }
6988
+ }
6989
+ function markdownLinks(value, base) {
6990
+ const links = [];
6991
+ const invalidLinks = [];
6992
+ for (const match of value.matchAll(/\[([^\u005d]+)\]\(([^)\s]+)(?:\s+"[^"]*")?\)/gu)) {
6993
+ if (!match[1] || !match[2]) continue;
6994
+ try {
6995
+ links.push({
6996
+ label: match[1],
6997
+ url: new URL(match[2], base).toString()
6998
+ });
6999
+ } catch {
7000
+ invalidLinks.push(match[2]);
7001
+ }
7002
+ }
7003
+ return {
7004
+ links,
7005
+ invalidLinks
7006
+ };
7007
+ }
7008
+ async function inspectLlmsTxt(input) {
7009
+ const url = new URL("/llms.txt", input.origin).toString();
7010
+ try {
7011
+ const [first, second] = await Promise.all([fetchText({
7012
+ url,
7013
+ timeoutMs: input.timeoutMs,
7014
+ fetch: input.fetch,
7015
+ signal: input.signal,
7016
+ accept: "text/plain,text/markdown;q=0.9"
7017
+ }), fetchText({
7018
+ url,
7019
+ timeoutMs: input.timeoutMs,
7020
+ fetch: input.fetch,
7021
+ signal: input.signal,
7022
+ accept: "text/plain,text/markdown;q=0.9"
7023
+ })]);
7024
+ const exists = first.response.status >= 200 && first.response.status < 300;
7025
+ const parsed = markdownLinks(first.body, url);
7026
+ const links = parsed.links.slice(0, MAX_LLMS_LINKS);
7027
+ const linkLimitReached = parsed.links.length > links.length;
7028
+ const counts = /* @__PURE__ */ new Map();
7029
+ for (const link of links) counts.set(link.url, (counts.get(link.url) ?? 0) + 1);
7030
+ const duplicateLinks = [...counts].filter(([, count]) => count > 1).map(([link]) => link).sort();
7031
+ const crawlRoutes = new Set(input.pages.map((page) => normalizedDocumentUrl(page.finalUrl)));
7032
+ const linkObservations = await Promise.all(links.map(async (link) => {
7033
+ const sameOrigin = new URL(link.url).origin === new URL(input.origin).origin;
7034
+ try {
7035
+ const target = await fetchText({
7036
+ url: link.url,
7037
+ timeoutMs: input.timeoutMs,
7038
+ fetch: input.fetch,
7039
+ signal: input.signal
7040
+ });
7041
+ const indexableTarget = sameOrigin && target.response.status >= 200 && target.response.status < 300 ? !responseIsNoindex(target.response, target.body) : void 0;
7042
+ const finalUrl = target.response.url || void 0;
7043
+ return {
7044
+ ...link,
7045
+ sameOrigin,
7046
+ status: target.response.status,
7047
+ finalUrl,
7048
+ redirected: target.response.redirected,
7049
+ ...indexableTarget === void 0 ? {} : { indexableTarget }
7050
+ };
7051
+ } catch (error) {
7052
+ return {
7053
+ ...link,
7054
+ sameOrigin,
7055
+ redirected: false,
7056
+ error: safeError(error)
7057
+ };
7058
+ }
7059
+ }));
7060
+ const linkedCrawlRoutes = new Set(linkObservations.filter((link) => link.sameOrigin && !new URL(link.url).pathname.startsWith("/.well-known/") && !new URL(link.url).pathname.endsWith(".md")).map((link) => normalizedDocumentUrl(link.url)));
7061
+ return {
7062
+ url,
7063
+ exists,
7064
+ status: first.response.status,
7065
+ contentType: first.response.headers.get("content-type") ?? void 0,
7066
+ bytes: Buffer.byteLength(first.body),
7067
+ sha256: sha256(first.body),
7068
+ repeatedHashStable: sha256(first.body) === sha256(second.body),
7069
+ headingCount: first.body.match(/^#{1,2}\s+\S/gmu)?.length ?? 0,
7070
+ totalParsedLinks: parsed.links.length,
7071
+ linkLimitReached,
7072
+ links: linkObservations,
7073
+ invalidLinks: parsed.invalidLinks.sort(),
7074
+ duplicateLinks,
7075
+ offSiteLinks: linkObservations.filter((link) => !link.sameOrigin).map((link) => link.url).sort(),
7076
+ redirectedLinks: linkObservations.filter((link) => link.redirected).map((link) => link.url).sort(),
7077
+ nonIndexableLinks: linkObservations.filter((link) => link.indexableTarget === false).map((link) => link.url).sort(),
7078
+ missingCrawlRoutes: [...linkedCrawlRoutes].filter((route) => !crawlRoutes.has(route)).sort(),
7079
+ oversized: Buffer.byteLength(first.body) > MAX_CURATED_LLMS_BYTES
7080
+ };
7081
+ } catch (error) {
7082
+ return {
7083
+ url,
7084
+ exists: false,
7085
+ repeatedHashStable: null,
7086
+ headingCount: 0,
7087
+ totalParsedLinks: 0,
7088
+ linkLimitReached: false,
7089
+ links: [],
7090
+ invalidLinks: [],
7091
+ duplicateLinks: [],
7092
+ offSiteLinks: [],
7093
+ redirectedLinks: [],
7094
+ nonIndexableLinks: [],
7095
+ missingCrawlRoutes: [],
7096
+ oversized: false,
7097
+ error: safeError(error)
7098
+ };
7099
+ }
7100
+ }
7101
+ async function inspectRouteManifest(input) {
7102
+ const url = new URL("/agent-routes.json", input.origin).toString();
7103
+ try {
7104
+ const result = await fetchText({
7105
+ url,
7106
+ timeoutMs: input.timeoutMs,
7107
+ fetch: input.fetch,
7108
+ signal: input.signal,
7109
+ accept: "application/json"
7110
+ });
7111
+ const parsed = JSON.parse(result.body);
7112
+ const declared = Array.isArray(parsed.pages) ? parsed.pages : [];
7113
+ const declaredHtmlRoutes = declared.map((page) => page.htmlPath).filter((path) => typeof path === "string").sort();
7114
+ const declaredMarkdownRoutes = declared.map((page) => page.markdownPath).filter((path) => typeof path === "string").sort();
7115
+ const crawledHtmlRoutes = new Set(input.pages.map((page) => new URL(page.finalUrl).pathname.replace(/\/$/u, "") || "/"));
7116
+ const observedMarkdownRoutes = new Set(input.observations.flatMap((page) => page.advertisedUrls.map((value) => new URL(value).pathname)));
7117
+ return {
7118
+ url,
7119
+ status: result.response.status,
7120
+ valid: Array.isArray(parsed.pages),
7121
+ declaredHtmlRoutes,
7122
+ declaredMarkdownRoutes,
7123
+ missingHtmlRoutes: declaredHtmlRoutes.filter((path) => !crawledHtmlRoutes.has(path.replace(/\/$/u, "") || "/")).sort(),
7124
+ missingMarkdownRoutes: declaredMarkdownRoutes.filter((path) => !observedMarkdownRoutes.has(path)).sort(),
7125
+ orphanMarkdownRoutes: [...observedMarkdownRoutes].filter((path) => !declaredMarkdownRoutes.includes(path)).sort()
7126
+ };
7127
+ } catch (error) {
7128
+ return {
7129
+ url,
7130
+ valid: false,
7131
+ declaredHtmlRoutes: [],
7132
+ declaredMarkdownRoutes: [],
7133
+ missingHtmlRoutes: [],
7134
+ missingMarkdownRoutes: [],
7135
+ orphanMarkdownRoutes: [],
7136
+ error: safeError(error)
7137
+ };
7138
+ }
7139
+ }
7140
+ async function inspectProtocolVariant(input) {
7141
+ try {
7142
+ const result = await fetchText({
7143
+ url: input.url,
7144
+ timeoutMs: input.timeoutMs,
7145
+ fetch: input.fetch,
7146
+ signal: input.signal,
7147
+ redirect: "manual"
7148
+ });
7149
+ return {
7150
+ status: result.response.status,
7151
+ location: result.response.headers.get("location") ?? void 0
7152
+ };
7153
+ } catch (error) {
7154
+ return { error: safeError(error) };
7155
+ }
7156
+ }
7157
+ async function collectAgentDiscovery(input) {
7158
+ const origin = new URL(input.startUrl).origin;
7159
+ const pages = input.pages.filter((page) => page.status >= 200 && page.status < 300 && /^\s*(?:text\/html|application\/xhtml\+xml)\b/iu.test(page.contentType ?? "")).sort((left, right) => left.finalUrl.localeCompare(right.finalUrl));
7160
+ const queue = new PQueue({ concurrency: input.concurrency ?? 4 });
7161
+ const observations = await Promise.all(pages.map((page) => queue.add(() => inspectMarkdownPage({
7162
+ page,
7163
+ timeoutMs: input.timeoutMs,
7164
+ fetch: input.fetch,
7165
+ signal: input.signal
7166
+ }))));
7167
+ const [agentSkills, llmsTxt, routeManifest] = await Promise.all([
7168
+ inspectAgentSkills({
7169
+ ...input,
7170
+ origin
7171
+ }),
7172
+ inspectLlmsTxt({
7173
+ ...input,
7174
+ origin,
7175
+ pages
7176
+ }),
7177
+ inspectRouteManifest({
7178
+ ...input,
7179
+ origin,
7180
+ pages,
7181
+ observations
7182
+ })
7183
+ ]);
7184
+ const qZero = await fetchRepresentation({
7185
+ url: input.startUrl,
7186
+ accept: "text/markdown;q=0,text/html;q=1",
7187
+ timeoutMs: input.timeoutMs,
7188
+ fetch: input.fetch,
7189
+ signal: input.signal
7190
+ });
7191
+ const start = new URL(input.startUrl);
7192
+ const httpUrl = new URL(start);
7193
+ httpUrl.protocol = "http:";
7194
+ const wwwUrl = new URL(start);
7195
+ wwwUrl.hostname = start.hostname.startsWith("www.") ? start.hostname.slice(4) : `www.${start.hostname}`;
7196
+ const [http, www] = await Promise.all([inspectProtocolVariant({
7197
+ ...input,
7198
+ url: httpUrl.toString()
7199
+ }), inspectProtocolVariant({
7200
+ ...input,
7201
+ url: wwwUrl.toString()
7202
+ })]);
7203
+ const httpLocation = http.location ? new URL(http.location, httpUrl).toString() : void 0;
7204
+ const wwwLocation = www.location ? new URL(www.location, wwwUrl).toString() : void 0;
7205
+ const qZeroContentType = qZero.observation.contentType;
7206
+ const advertisedPages = observations.filter((observation) => observation.htmlAlternateUnique).length;
7207
+ const evaluatedPages = observations.filter((observation) => observation.explicit?.status !== void 0).length;
7208
+ const warnings = [...pages.length !== input.pages.length ? [`Agent representations were evaluated for ${pages.length} successful HTML pages from ${input.pages.length} retained documents.`] : [], ...routeManifest.valid ? [] : ["No valid public agent route manifest was available for orphan checks."]].sort();
7209
+ const htmlContentSignals = pages.map((page) => headerValue$1(page.responseHeaders, "content-signal")).filter((value) => Boolean(value));
7210
+ const markdownContentSignals = observations.map((observation) => observation.explicit?.contentSignal).filter((value) => Boolean(value));
7211
+ const contentSignalValues = [.../* @__PURE__ */ new Set([...htmlContentSignals, ...markdownContentSignals])].sort();
7212
+ return {
7213
+ profile: "content",
7214
+ profileApplicability: {
7215
+ content: {
7216
+ status: "evaluated",
7217
+ reason: "This site publishes documents, so access, representations, discovery, identity, and extraction quality were evaluated."
7218
+ },
7219
+ api: {
7220
+ status: "notApplicable",
7221
+ reason: "No public API surface was selected for this content-site run."
7222
+ },
7223
+ application: {
7224
+ status: "notApplicable",
7225
+ reason: "A local MCP product does not make this website a public remote agent endpoint."
7226
+ },
7227
+ commerce: {
7228
+ status: "notApplicable",
7229
+ reason: "No product checkout or transactional surface was selected."
7230
+ }
7231
+ },
7232
+ dataStatus: pages.length === 0 ? "unavailable" : evaluatedPages === pages.length ? "complete" : "partial",
7233
+ markdownAlternates: {
7234
+ eligibleHtmlPages: pages.length,
7235
+ advertisedPages,
7236
+ evaluatedPages,
7237
+ exactByteMatches: observations.filter((observation) => observation.explicitMatchesNegotiated).length,
7238
+ stableResponses: observations.filter((observation) => observation.repeatedHashStable).length,
7239
+ pages: observations
7240
+ },
7241
+ contentNegotiation: {
7242
+ qZeroHonoured: qZeroContentType !== void 0 ? !/^\s*text\/markdown\b/iu.test(qZeroContentType) : null,
7243
+ qZeroStatus: qZero.observation.status,
7244
+ qZeroContentType,
7245
+ error: qZero.observation.error
7246
+ },
7247
+ routeManifest,
7248
+ agentSkills,
7249
+ llmsTxt,
7250
+ contentSignals: {
7251
+ htmlValues: [...new Set(htmlContentSignals)].sort(),
7252
+ markdownValues: [...new Set(markdownContentSignals)].sort(),
7253
+ missingHtmlPages: pages.length - htmlContentSignals.length,
7254
+ missingMarkdownPages: observations.length - markdownContentSignals.length,
7255
+ consistent: pages.length > 0 && observations.length > 0 ? contentSignalValues.length === 1 && htmlContentSignals.length === pages.length && markdownContentSignals.length === observations.length : null
7256
+ },
7257
+ protocolVariants: {
7258
+ http: {
7259
+ url: httpUrl.toString(),
7260
+ status: http.status,
7261
+ location: httpLocation,
7262
+ permanentRedirectToHttps: http.status !== void 0 ? [301, 308].includes(http.status) && Boolean(httpLocation?.startsWith("https://")) : null,
7263
+ error: http.error
7264
+ },
7265
+ www: {
7266
+ url: wwwUrl.toString(),
7267
+ status: www.status,
7268
+ location: wwwLocation,
7269
+ redirectsToPreferredHost: www.status !== void 0 ? [301, 308].includes(www.status) && Boolean(wwwLocation && new URL(wwwLocation).hostname === start.hostname && new URL(wwwLocation).protocol === "https:") : null,
7270
+ error: www.error
7271
+ },
7272
+ hstsOnStartPage: pages.find((page) => normalizedDocumentUrl(page.finalUrl) === normalizedDocumentUrl(input.startUrl))?.hasHsts ?? null
7273
+ },
7274
+ warnings
7275
+ };
7276
+ }
7277
+ //#endregion
7278
+ //#region packages/core/src/analyze/crawler/agent-readiness.ts
7279
+ function check$1(section, input) {
7280
+ return {
7281
+ section,
7282
+ ...input
7283
+ };
7284
+ }
7285
+ function section$1(id, title, checks) {
7286
+ return {
7287
+ id,
7288
+ title,
7289
+ checks
7290
+ };
7291
+ }
7292
+ function failedMarkdownUrls(discovery) {
7293
+ return discovery.markdownAlternates.pages.filter((page) => !page.htmlAlternateUnique || page.httpAlternateUrls.length !== 1 || page.explicit?.status !== 200 || !/^\s*text\/markdown\b/iu.test(page.explicit.contentType ?? "") || page.explicitMatchesNegotiated !== true || page.repeatedHashStable !== true || page.markdownCanonicalMatchesHtml !== true || page.explicit?.varyAccept !== true || page.negotiated?.varyAccept !== true).map((page) => page.htmlUrl);
7294
+ }
7295
+ function qualityFailureUrls(discovery) {
7296
+ return discovery.markdownAlternates.pages.filter((page) => {
7297
+ const quality = page.quality;
7298
+ if (!quality) return true;
7299
+ return !quality.frontmatterTitle || quality.h1Count !== 1 || !quality.codeFenceBalanced || quality.rawHtmlTags > 0 || quality.rawSvgTags > 0 || quality.rawScriptTags > 0 || quality.rawStyleTags > 0 || quality.navigationOnly || quality.repeatedLines > 0 || quality.wordRetentionRatio !== null && quality.wordRetentionRatio < .4;
7300
+ }).map((page) => page.htmlUrl);
7301
+ }
7302
+ function profileChecks(discovery) {
7303
+ return [
7304
+ "api",
7305
+ "application",
7306
+ "commerce"
7307
+ ].map((profile) => check$1("profile", {
7308
+ id: `profile-${profile}`,
7309
+ status: "notApplicable",
7310
+ title: `${profile[0]?.toUpperCase()}${profile.slice(1)} checks are not applicable`,
7311
+ plainEnglish: discovery.profileApplicability[profile].reason,
7312
+ action: "Choose this profile only when the site exposes that capability publicly.",
7313
+ evidence: {
7314
+ profile,
7315
+ selectedProfile: discovery.profile
7316
+ }
7317
+ }));
7318
+ }
7319
+ function summariseChecks(checks) {
7320
+ const count = (status) => checks.filter((item) => item.status === status).length;
7321
+ return {
7322
+ checks: checks.length,
7323
+ passed: count("pass"),
7324
+ warnings: count("warning"),
7325
+ failed: count("fail"),
7326
+ unknown: count("unknown"),
7327
+ information: count("info"),
7328
+ notApplicable: count("notApplicable")
7329
+ };
7330
+ }
7331
+ function markdownChecks(discovery) {
7332
+ const markdown = discovery.markdownAlternates;
7333
+ const failedUrls = failedMarkdownUrls(discovery);
7334
+ const qualityUrls = qualityFailureUrls(discovery);
7335
+ const coverageComplete = markdown.eligibleHtmlPages > 0 && markdown.advertisedPages === markdown.eligibleHtmlPages && markdown.evaluatedPages === markdown.eligibleHtmlPages;
7336
+ const bytesMatch = markdown.eligibleHtmlPages > 0 && markdown.exactByteMatches === markdown.eligibleHtmlPages;
7337
+ const stable = markdown.eligibleHtmlPages > 0 && markdown.stableResponses === markdown.eligibleHtmlPages;
7338
+ const tokenHeaders = markdown.pages.filter((page) => page.explicit?.bytes !== void 0 && page.explicit?.markdownTokens !== void 0 && page.explicit.markdownTokens === Math.ceil(page.explicit.bytes / 4) && page.explicit.markdownTokens === page.negotiated?.markdownTokens).length;
7339
+ return [
7340
+ check$1("representations", {
7341
+ id: "markdown-coverage",
7342
+ status: markdown.eligibleHtmlPages === 0 ? "unknown" : coverageComplete ? "pass" : "fail",
7343
+ title: coverageComplete ? "Every successful HTML page advertises one Markdown alternative" : "Markdown alternative coverage is incomplete",
7344
+ plainEnglish: `${markdown.advertisedPages} of ${markdown.eligibleHtmlPages} successful HTML pages advertised one Markdown alternative, and ${markdown.evaluatedPages} were fetched.`,
7345
+ action: coverageComplete ? "Keep route generation tied to the same deterministic page manifest." : "Add one text/markdown alternate for each public HTML page and make every advertised URL return successfully.",
7346
+ evidence: {
7347
+ eligibleHtmlPages: markdown.eligibleHtmlPages,
7348
+ advertisedPages: markdown.advertisedPages,
7349
+ evaluatedPages: markdown.evaluatedPages
7350
+ },
7351
+ urls: failedUrls.slice(0, 25)
7352
+ }),
7353
+ check$1("representations", {
7354
+ id: "markdown-token-estimates",
7355
+ status: markdown.evaluatedPages === 0 ? "unknown" : tokenHeaders === markdown.evaluatedPages ? "pass" : "warning",
7356
+ title: tokenHeaders === markdown.evaluatedPages ? "Every Markdown response includes the same token estimate" : "Some Markdown responses are missing a stable token estimate",
7357
+ plainEnglish: `${tokenHeaders} of ${markdown.evaluatedPages} evaluated pages returned matching X-Markdown-Tokens values on the explicit and negotiated responses. The value is a quick size estimate, not an exact model-specific token count.`,
7358
+ action: "Generate the estimate from the stored Markdown bytes and return the same value for explicit and negotiated responses.",
7359
+ urls: markdown.pages.filter((page) => page.explicit?.markdownTokens === void 0 || page.explicit.markdownTokens !== page.negotiated?.markdownTokens).map((page) => page.htmlUrl).slice(0, 25)
7360
+ }),
7361
+ check$1("representations", {
7362
+ id: "markdown-negotiation",
7363
+ status: markdown.eligibleHtmlPages === 0 ? "unknown" : bytesMatch && discovery.contentNegotiation.qZeroHonoured ? "pass" : "fail",
7364
+ title: bytesMatch && discovery.contentNegotiation.qZeroHonoured ? "Explicit and negotiated Markdown use the same bytes" : "Content negotiation does not match the explicit Markdown contract",
7365
+ plainEnglish: `${markdown.exactByteMatches} of ${markdown.eligibleHtmlPages} pages returned identical explicit and negotiated Markdown. A request that refuses Markdown ${discovery.contentNegotiation.qZeroHonoured ? "received HTML" : "did not produce confirmed HTML evidence"}.`,
7366
+ action: "Use one stored Markdown artifact for both routes, honour Accept q-values, and send Vary: Accept on negotiated responses.",
7367
+ evidence: {
7368
+ exactByteMatches: markdown.exactByteMatches,
7369
+ qZero: discovery.contentNegotiation
7370
+ },
7371
+ urls: failedUrls.slice(0, 25)
7372
+ }),
7373
+ check$1("representations", {
7374
+ id: "markdown-determinism",
7375
+ status: markdown.eligibleHtmlPages === 0 ? "unknown" : stable ? "pass" : "fail",
7376
+ title: stable ? "Repeated Markdown responses are stable" : "Repeated Markdown responses changed during the audit",
7377
+ plainEnglish: `${markdown.stableResponses} of ${markdown.eligibleHtmlPages} repeated explicit responses kept the same SHA-256 digest.`,
7378
+ action: "Generate Markdown during the build and remove runtime rewriting, timestamps, or random output.",
7379
+ evidence: { stableResponses: markdown.stableResponses },
7380
+ urls: failedUrls.slice(0, 25)
7381
+ }),
7382
+ check$1("representations", {
7383
+ id: "markdown-quality",
7384
+ status: markdown.evaluatedPages === 0 ? "unknown" : qualityUrls.length === 0 ? "pass" : "warning",
7385
+ title: qualityUrls.length === 0 ? "Markdown keeps the useful document structure cleanly" : "Some Markdown alternatives need an extraction review",
7386
+ plainEnglish: qualityUrls.length === 0 ? "The evaluated alternatives retained one H1, frontmatter titles, balanced code fences, useful copy, and no leaked SVG, script, style, or layout tags." : `${qualityUrls.length} Markdown alternative${qualityUrls.length === 1 ? "" : "s"} lost important structure, looked navigation-only, or leaked presentation markup.`,
7387
+ action: "Open the affected Markdown directly and fix the shared converter or component exclusion rules rather than patching generated files.",
7388
+ urls: qualityUrls.slice(0, 25)
7389
+ })
7390
+ ];
7391
+ }
7392
+ function discoveryChecks(discovery) {
7393
+ const skills = discovery.agentSkills;
7394
+ const skillsValid = skills.validIndex && skills.skills.length > 0 && skills.skills.every((skill) => skill.sameOrigin && skill.status === 200 && /^\s*text\/markdown\b/iu.test(skill.contentType ?? "") && Boolean(skill.cors) && skill.digestMatches === true && skill.frontmatterValid === true) && Boolean(skills.cors);
7395
+ const llms = discovery.llmsTxt;
7396
+ const llmsBroken = llms.links.filter((link) => !link.status || link.status < 200 || link.status >= 400);
7397
+ const llmsValid = llms.exists && /^\s*(?:text\/plain|text\/markdown)\b/iu.test(llms.contentType ?? "") && llms.headingCount > 0 && llms.links.length > 0 && !llms.linkLimitReached && !llms.oversized && llms.invalidLinks.length === 0 && llms.duplicateLinks.length === 0 && llms.redirectedLinks.length === 0 && llms.nonIndexableLinks.length === 0 && llms.missingCrawlRoutes.length === 0 && llmsBroken.length === 0 && llms.repeatedHashStable === true;
7398
+ return [
7399
+ check$1("discovery", {
7400
+ id: "agent-skills",
7401
+ status: skillsValid ? "pass" : skills.status === 404 ? "info" : "warning",
7402
+ title: skillsValid ? "Agent Skills discovery is valid and digest verified" : "Agent Skills discovery is absent or needs attention",
7403
+ plainEnglish: skillsValid ? `${skills.skills.length} published skill${skills.skills.length === 1 ? "" : "s"} matched the declared SHA-256 digest, valid SKILL.md frontmatter, Markdown content type, and cross-origin access policy.` : "Agent Skills is optional for a content site, but a published index should resolve same-origin Markdown skill files, allow intended cross-origin clients, and verify their declared bytes.",
7404
+ action: "If you publish Agent Skills, keep the discovery index and SHA-256 digests generated from the exact deployed skill bytes.",
7405
+ evidence: skills
7406
+ }),
7407
+ check$1("discovery", {
7408
+ id: "llms-txt",
7409
+ status: llms.exists ? llmsValid ? "pass" : "warning" : "info",
7410
+ title: llms.exists ? llmsValid ? "llms.txt is short, stable, and its links resolve" : "llms.txt exists but needs a content or link review" : "llms.txt is not published",
7411
+ plainEnglish: llms.exists ? `${llms.links.length} declared links were checked. ${llms.invalidLinks.length} were malformed, ${llms.duplicateLinks.length} were duplicated, ${llms.redirectedLinks.length} redirected, ${llms.nonIndexableLinks.length} reached non-indexable pages, ${llmsBroken.length} did not resolve, and ${llms.missingCrawlRoutes.length} were missing from the crawl inventory. ${llms.offSiteLinks.length} linked to other sites.` : "llms.txt is optional and its absence is not a search ranking problem.",
7412
+ action: llms.exists ? "Keep the file curated, deterministic, and limited to useful entry points whose links still resolve." : "Add it only when an intended consumer uses it. Do not treat it as a Google ranking requirement.",
7413
+ evidence: {
7414
+ status: llms.status,
7415
+ contentType: llms.contentType,
7416
+ bytes: llms.bytes,
7417
+ oversized: llms.oversized,
7418
+ totalParsedLinks: llms.totalParsedLinks,
7419
+ linkLimitReached: llms.linkLimitReached,
7420
+ links: llms.links.length,
7421
+ invalidLinks: llms.invalidLinks,
7422
+ duplicateLinks: llms.duplicateLinks,
7423
+ offSiteLinks: llms.offSiteLinks,
7424
+ redirectedLinks: llms.redirectedLinks,
7425
+ nonIndexableLinks: llms.nonIndexableLinks,
7426
+ missingCrawlRoutes: llms.missingCrawlRoutes,
7427
+ repeatedHashStable: llms.repeatedHashStable
7428
+ },
7429
+ urls: [
7430
+ ...llmsBroken.map((link) => link.url),
7431
+ ...llms.redirectedLinks,
7432
+ ...llms.nonIndexableLinks,
7433
+ ...llms.missingCrawlRoutes
7434
+ ].slice(0, 25)
7435
+ }),
7436
+ check$1("discovery", {
7437
+ id: "content-signals",
7438
+ status: discovery.contentSignals.consistent === null ? "unknown" : discovery.contentSignals.consistent ? "pass" : "warning",
7439
+ title: discovery.contentSignals.consistent ? "HTML and Markdown publish one explicit Content Signals policy" : "Content Signals are missing or inconsistent across representations",
7440
+ plainEnglish: discovery.contentSignals.consistent ? `The same ${discovery.contentSignals.htmlValues[0]} policy was observed on every evaluated HTML and Markdown response.` : `${discovery.contentSignals.missingHtmlPages} HTML responses and ${discovery.contentSignals.missingMarkdownPages} Markdown responses were missing the header, or the observed values differed.`,
7441
+ action: "Choose the site policy deliberately and return the same Content-Signal value on HTML and Markdown responses.",
7442
+ evidence: discovery.contentSignals
7443
+ }),
7444
+ check$1("discovery", {
7445
+ id: "route-manifest",
7446
+ status: !discovery.routeManifest.valid ? "info" : discovery.routeManifest.missingHtmlRoutes.length === 0 && discovery.routeManifest.missingMarkdownRoutes.length === 0 && discovery.routeManifest.orphanMarkdownRoutes.length === 0 ? "pass" : "warning",
7447
+ title: discovery.routeManifest.valid ? "The public route manifest agrees with the crawl" : "No public route manifest was available",
7448
+ plainEnglish: discovery.routeManifest.valid ? `The manifest declared ${discovery.routeManifest.declaredHtmlRoutes.length} HTML routes and ${discovery.routeManifest.declaredMarkdownRoutes.length} Markdown routes. Missing or orphan routes are listed in the evidence.` : "A public route manifest is optional. Without one, this crawl cannot prove that no orphan Markdown files were deployed.",
7449
+ action: "Generate one deterministic route inventory from the build when exact HTML and Markdown parity matters.",
7450
+ evidence: discovery.routeManifest
7451
+ })
7452
+ ];
7453
+ }
7454
+ function accessChecks(report, discovery) {
7455
+ const bots = report.ai?.robotsTxt?.botAccess ?? [];
7456
+ const blocked = bots.filter((bot) => bot.allowed === false);
7457
+ const robotsKnown = report.ai?.robotsTxt?.availability === "available" || report.ai?.robotsTxt?.availability === "absent";
7458
+ const protocol = discovery.protocolVariants;
7459
+ return [check$1("access", {
7460
+ id: "crawler-access",
7461
+ status: !robotsKnown ? "unknown" : blocked.length > 0 ? "warning" : "pass",
7462
+ title: blocked.length ? "robots.txt blocks selected crawler tokens" : "No selected crawler token is blocked at the start URL",
7463
+ plainEnglish: robotsKnown ? `${blocked.length} of ${bots.length} selected crawler tokens were blocked by the start URL robots.txt policy.` : "The robots.txt response was not stable enough to make a crawler-access claim.",
7464
+ action: "Keep blocks intentional. A crawler token being allowed here does not prove that the service will crawl, index, or cite the site.",
7465
+ evidence: {
7466
+ availability: report.ai?.robotsTxt?.availability,
7467
+ blocked,
7468
+ botAccess: bots
7469
+ }
7470
+ }), check$1("access", {
7471
+ id: "protocol-canonicalization",
7472
+ status: protocol.http.permanentRedirectToHttps === true && protocol.hstsOnStartPage === true ? "pass" : protocol.http.permanentRedirectToHttps === null ? "unknown" : "warning",
7473
+ title: protocol.http.permanentRedirectToHttps === true && protocol.hstsOnStartPage === true ? "HTTP redirects permanently to HTTPS and HTTPS sends HSTS" : "Protocol canonicalization needs attention or could not be proven",
7474
+ plainEnglish: `The HTTP variant ${protocol.http.permanentRedirectToHttps === true ? "redirected permanently to HTTPS" : "did not produce a confirmed permanent HTTPS redirect"}. The HTTPS start page ${protocol.hstsOnStartPage ? "sent HSTS" : "did not provide confirmed HSTS evidence"}.`,
7475
+ action: "Use one permanent HTTP-to-HTTPS redirect and send HSTS only after HTTPS is stable across the site.",
7476
+ evidence: protocol
7477
+ })];
7478
+ }
7479
+ function identityChecks(report) {
7480
+ const types = new Set(report.pages.flatMap((page) => page.schemaTypes ?? []));
7481
+ const hasSoftware = types.has("SoftwareApplication");
7482
+ const hasWebsite = types.has("WebSite");
7483
+ const hasCreator = types.has("Person") || types.has("Organization");
7484
+ const pageTypes = [
7485
+ "WebPage",
7486
+ "TechArticle",
7487
+ "CollectionPage"
7488
+ ].filter((type) => types.has(type));
7489
+ const officialProfiles = [...new Set(report.pages.flatMap((page) => [...page.schemaSameAs ?? [], ...page.socialProfileLinks ?? []]))];
7490
+ return [check$1("identity", {
7491
+ id: "identity-graph",
7492
+ status: hasSoftware && hasWebsite && hasCreator && pageTypes.length > 0 ? "pass" : "warning",
7493
+ title: hasSoftware && hasWebsite && hasCreator && pageTypes.length > 0 ? "Structured identity covers the software, website, creator, and pages" : "The structured identity graph is missing a useful entity layer",
7494
+ plainEnglish: `Software identity: ${hasSoftware ? "present" : "missing"}. Website identity: ${hasWebsite ? "present" : "missing"}. Creator or publisher identity: ${hasCreator ? "present" : "missing"}. Page-level types: ${pageTypes.length ? pageTypes.join(", ") : "missing"}.`,
7495
+ action: "Connect truthful SoftwareApplication, WebSite, creator or publisher, and page nodes. Use sameAs only for official profiles that identify the same entity.",
7496
+ evidence: {
7497
+ schemaTypes: [...types].sort(),
7498
+ officialProfiles: officialProfiles.sort()
7499
+ }
7500
+ })];
7501
+ }
7502
+ function agentReadiness(report) {
7503
+ const discovery = report.agentDiscovery;
7504
+ if (!discovery) {
7505
+ const profiles = {
7506
+ content: {
7507
+ status: "evaluated",
7508
+ reason: "The content profile was selected but discovery evidence was not collected."
7509
+ },
7510
+ api: {
7511
+ status: "notApplicable",
7512
+ reason: "Not selected."
7513
+ },
7514
+ application: {
7515
+ status: "notApplicable",
7516
+ reason: "Not selected."
7517
+ },
7518
+ commerce: {
7519
+ status: "notApplicable",
7520
+ reason: "Not selected."
7521
+ }
7522
+ };
7523
+ const missing = check$1("collection", {
7524
+ id: "agent-discovery-evidence",
7525
+ status: "unknown",
7526
+ title: "Agent discovery evidence was not collected",
7527
+ plainEnglish: "This saved crawl predates the focused content-profile checks or was run without them.",
7528
+ action: "Run the agent-readiness report with a URL or start a fresh crawl with agent discovery enabled."
7529
+ });
7530
+ const scopeChecks = profileChecks({
7531
+ profile: "content",
7532
+ profileApplicability: profiles
7533
+ });
7534
+ const checks = [missing, ...scopeChecks];
7535
+ return {
7536
+ reportId: report.id,
7537
+ url: report.config.url,
7538
+ generatedAt: report.generatedAt,
7539
+ profile: "content",
7540
+ dataStatus: "unavailable",
7541
+ assessment: "evidence-only",
7542
+ headline: "No agent-discovery evidence is stored in this crawl. Run a fresh focused check.",
7543
+ profileApplicability: profiles,
7544
+ summary: summariseChecks(checks),
7545
+ sections: [section$1("collection", "Evidence collection", [missing]), section$1("profile", "Profile scope", scopeChecks)],
7546
+ checks,
7547
+ topActions: [missing],
7548
+ caveats: ["Not evaluated does not mean absent or broken.", "This report does not measure rankings, AI mentions, citations, or selection."]
7549
+ };
7550
+ }
7551
+ const sections = [
7552
+ section$1("profile", "Profile scope", profileChecks(discovery)),
7553
+ section$1("access", "Crawler and protocol access", accessChecks(report, discovery)),
7554
+ section$1("representations", "HTML and Markdown representations", markdownChecks(discovery)),
7555
+ section$1("discovery", "Agent discovery files", discoveryChecks(discovery)),
7556
+ section$1("identity", "Identity evidence", identityChecks(report))
7557
+ ];
7558
+ const checks = sections.flatMap((item) => item.checks);
7559
+ const actionable = checks.filter((item) => [
7560
+ "fail",
7561
+ "warning",
7562
+ "unknown"
7563
+ ].includes(item.status));
7564
+ const rank = {
7565
+ fail: 0,
7566
+ warning: 1,
7567
+ unknown: 2,
7568
+ pass: 3,
7569
+ info: 4,
7570
+ notApplicable: 5
7571
+ };
7572
+ const topActions = [...actionable].sort((left, right) => rank[left.status] - rank[right.status]).slice(0, 10);
7573
+ const summary = summariseChecks(checks);
7574
+ return {
7575
+ reportId: report.id,
7576
+ url: report.config.url,
7577
+ generatedAt: report.generatedAt,
7578
+ profile: discovery.profile,
7579
+ dataStatus: discovery.dataStatus,
7580
+ assessment: "evidence-only",
7581
+ headline: `${summary.passed} checks passed, ${summary.warnings} need review, and ${summary.failed} failed for the content profile.`,
7582
+ profileApplicability: discovery.profileApplicability,
7583
+ summary,
7584
+ sections,
7585
+ checks,
7586
+ topActions,
7587
+ caveats: [
7588
+ ...report.caveats,
7589
+ ...discovery.warnings,
7590
+ "A clean content profile does not prove indexing, rankings, AI mentions, citations, or selection.",
7591
+ "A crawler token being allowed does not prove that a service fetched or used the page.",
7592
+ "API, application, and commerce checks were not applicable to this content-site run."
7593
+ ]
7594
+ };
7595
+ }
7596
+ //#endregion
7597
+ //#region packages/core/src/analyze/crawler/entity-readiness.ts
7598
+ const ENTITY_EVIDENCE_SCHEMA_TYPES = /* @__PURE__ */ new Set([
7599
+ "organization",
7600
+ "localbusiness",
7601
+ "person",
7602
+ "product",
7603
+ "softwareapplication",
7604
+ "website"
7605
+ ]);
7606
+ const SITE_IDENTITY_SCHEMA_TYPES = /* @__PURE__ */ new Set(["organization", "localbusiness"]);
7607
+ const PROFILE_IDENTITY_SCHEMA_TYPES = /* @__PURE__ */ new Set([...SITE_IDENTITY_SCHEMA_TYPES, "person"]);
7608
+ function isEntityEvidenceSchemaType(type) {
7609
+ return ENTITY_EVIDENCE_SCHEMA_TYPES.has(type.toLowerCase());
7610
+ }
7611
+ function isSiteIdentitySchemaType(type) {
7612
+ return SITE_IDENTITY_SCHEMA_TYPES.has(type.toLowerCase());
7613
+ }
7614
+ function isProfileIdentitySchemaType(type) {
7615
+ return PROFILE_IDENTITY_SCHEMA_TYPES.has(type.toLowerCase());
7616
+ }
7617
+ function unique$3(values) {
7618
+ return [...new Set(values.filter((value) => Boolean(value)))];
7619
+ }
7620
+ function pct$3(part, total) {
7621
+ return total ? Math.round(part / total * 100) : 0;
7622
+ }
7623
+ function entityReadiness(report) {
7624
+ const pages = report.pages.filter((page) => page.indexable && page.status < 400);
7625
+ const total = pages.length || 1;
7626
+ const schemaTypes = {};
7627
+ for (const page of pages) for (const type of page.schemaTypes ?? []) schemaTypes[type] = (schemaTypes[type] ?? 0) + 1;
7628
+ const entitySchemaPages = pages.filter((page) => page.schemaTypes?.some(isEntityEvidenceSchemaType));
7629
+ const sameAsEvidence = pages.flatMap((page) => page.schemaSameAsEvidence ?? []).filter((evidence) => evidence.subjectTypes.some(isProfileIdentitySchemaType));
7630
+ const siteSameAsEvidence = sameAsEvidence.filter((evidence) => evidence.subjectTypes.some(isSiteIdentitySchemaType));
7631
+ const sameAs = unique$3(sameAsEvidence.map((evidence) => evidence.url));
7632
+ const sameAsByType = {};
7633
+ for (const evidence of sameAsEvidence) for (const type of evidence.subjectTypes) {
7634
+ if (!isProfileIdentitySchemaType(type)) continue;
7635
+ sameAsByType[type] = unique$3([...sameAsByType[type] ?? [], evidence.url]);
7636
+ }
7637
+ const socialProfiles = unique$3(pages.flatMap((page) => page.socialProfileLinks ?? []));
7638
+ const authors = unique$3(pages.map((page) => page.author));
7639
+ const authorPages = pages.filter((page) => page.author || page.geo?.hasAuthor);
7640
+ const datePages = pages.filter((page) => page.hasDate || page.geo?.hasDate);
7641
+ const titlePages = pages.filter((page) => page.title && page.h1);
7642
+ const checks = [
7643
+ {
7644
+ coverage: pct$3(entitySchemaPages.length, total),
7645
+ check: {
7646
+ id: "entity-schema",
7647
+ title: "Entity schema is present",
7648
+ plainEnglish: `${pct$3(entitySchemaPages.length, total)}% of evaluated indexable pages include Organization, LocalBusiness, Person, Product, SoftwareApplication, or WebSite schema.`,
7649
+ action: "Add accurate schema only where it describes the software, website, organization, person, product, or local business shown on the page.",
7650
+ evidence: { schemaTypes },
7651
+ urls: pages.filter((page) => !entitySchemaPages.includes(page)).slice(0, 10).map((page) => page.finalUrl)
7652
+ }
7653
+ },
7654
+ {
7655
+ coverage: siteSameAsEvidence.length ? 100 : 0,
7656
+ check: {
7657
+ id: "same-as",
7658
+ title: "Site entity profiles are connected",
7659
+ plainEnglish: siteSameAsEvidence.length ? "The crawl found sameAs links attached to Organization or LocalBusiness structured data." : sameAs.length ? "The crawl found Person sameAs links, but no Organization or LocalBusiness profile evidence for the site entity." : socialProfiles.length ? "The crawl found social-domain links, but they are not enough to prove official profile ownership." : "The crawl did not find entity-scoped sameAs evidence.",
7660
+ action: "Connect official profiles with sameAs on Organization, Person, or LocalBusiness schema. Use only profiles you control.",
7661
+ evidence: {
7662
+ sameAs,
7663
+ siteSameAs: unique$3(siteSameAsEvidence.map((evidence) => evidence.url)),
7664
+ sameAsEvidence,
7665
+ unclassifiedSocialLinks: socialProfiles
7666
+ }
7667
+ }
7668
+ },
7669
+ {
7670
+ coverage: Math.round((pct$3(authorPages.length, total) + pct$3(datePages.length, total)) / 2),
7671
+ check: {
7672
+ id: "authority-freshness",
7673
+ title: "Author and date signals observed",
7674
+ plainEnglish: `${pct$3(authorPages.length, total)}% of evaluated indexable pages expose an author signal and ${pct$3(datePages.length, total)}% expose a date signal.`,
7675
+ action: "For editorial content, identify authors and show accurate published or modified dates when that information is useful to readers.",
7676
+ evidence: {
7677
+ authorCoverage: pct$3(authorPages.length, total),
7678
+ dateCoverage: pct$3(datePages.length, total),
7679
+ authors
7680
+ }
7681
+ }
7682
+ },
7683
+ {
7684
+ coverage: pct$3(titlePages.length, total),
7685
+ check: {
7686
+ id: "entity-naming",
7687
+ title: "Title and H1 coverage observed",
7688
+ plainEnglish: `${pct$3(titlePages.length, total)}% of evaluated indexable pages have both a title and H1.`,
7689
+ action: "Review pages missing either signal. Keep brand, product, and person names accurate wherever they appear."
7690
+ }
7691
+ }
7692
+ ].map(({ coverage, check }) => ({
7693
+ ...check,
7694
+ status: "info",
7695
+ evidence: {
7696
+ ...check.evidence,
7697
+ observedCoveragePercent: coverage
7698
+ }
7699
+ }));
7700
+ const dataStatus = report.status === "completed" && pages.length > 0 ? "complete" : "partial";
7701
+ return {
7702
+ reportId: report.id,
7703
+ url: report.config.url,
7704
+ generatedAt: report.generatedAt,
7705
+ dataStatus,
7706
+ evaluatedPages: pages.length,
7707
+ crawlPages: report.pages.length,
7708
+ assessment: "evidence-only",
7709
+ headline: dataStatus === "partial" ? "Entity evidence is incomplete; treat these findings as scoped to the evaluated pages, not the whole site." : "Entity signals are reported as observations, not a ranking, visibility, or machine-understanding score.",
7710
+ caveats: [
7711
+ ...report.caveats,
7712
+ "This report deliberately has no aggregate entity score. Add schema, authors, dates, and sameAs only where they accurately fit the page and a documented use case.",
7713
+ ...dataStatus === "partial" ? [`Entity coverage is based on ${pages.length} evaluated indexable page${pages.length === 1 ? "" : "s"} from a ${report.status} crawl.`] : [],
7714
+ ...socialProfiles.length && !sameAs.length ? ["Social-domain links are unclassified navigation evidence, not proof that the linked profiles are official."] : []
7715
+ ],
7716
+ checks,
7717
+ entities: {
7718
+ schemaTypes,
7719
+ sameAs,
7720
+ sameAsByType,
7721
+ socialProfiles,
7722
+ authors
6636
7723
  }
6637
7724
  };
6638
7725
  }
6639
- function matchesFilters(issue, filters) {
6640
- if (filters.ruleId && issue.ruleId !== filters.ruleId) return false;
6641
- if (filters.category && issue.category !== filters.category) return false;
6642
- if (filters.severity && issue.severity !== filters.severity) return false;
6643
- return true;
6644
- }
6645
7726
  //#endregion
6646
7727
  //#region packages/core/src/analyze/crawler/ai-readiness.ts
6647
- function pct$3(part, total) {
7728
+ function pct$2(part, total) {
6648
7729
  if (total <= 0) return 0;
6649
7730
  return Math.round(part / total * 100);
6650
7731
  }
@@ -6681,6 +7762,7 @@ function sampleUrls$1(pages, predicate) {
6681
7762
  return pages.filter(predicate).slice(0, 10).map((page) => page.finalUrl);
6682
7763
  }
6683
7764
  function aiReadiness(report) {
7765
+ const agentDiscovery = report.agentDiscovery;
6684
7766
  const pages = indexablePages$1(report);
6685
7767
  const pageCount = pages.length;
6686
7768
  const hasIndexablePages = pageCount > 0;
@@ -6704,7 +7786,7 @@ function aiReadiness(report) {
6704
7786
  const titlePages = pages.filter((page) => page.title && page.h1);
6705
7787
  const metaPages = pages.filter((page) => page.metaDescription);
6706
7788
  const langPages = pages.filter((page) => page.lang);
6707
- const entityPages = pages.filter((page) => page.schemaTypes?.some((type) => /^(Organization|LocalBusiness|Person|Product|WebSite)$/i.test(type)) || (page.schemaSameAs?.length ?? 0) > 0 || (page.socialProfileLinks?.length ?? 0) > 0);
7789
+ const entityPages = pages.filter((page) => page.schemaTypes?.some(isEntityEvidenceSchemaType) || (page.schemaSameAs?.length ?? 0) > 0 || (page.socialProfileLinks?.length ?? 0) > 0);
6708
7790
  const restrictedSnippets = pages.map((page) => ({
6709
7791
  url: page.finalUrl,
6710
7792
  control: effectiveSnippetControl({
@@ -6721,7 +7803,7 @@ function aiReadiness(report) {
6721
7803
  section: "agent-access",
6722
7804
  maxScore: 20,
6723
7805
  evaluated: botAccessEvaluated,
6724
- score: robotsUnavailable ? 0 : botAccess.length ? blockedBots.length ? scoreFromPercent(pct$3(botAccess.length - blockedBots.length, botAccess.length), 20) : 20 : 8,
7806
+ score: robotsUnavailable ? 0 : botAccess.length ? blockedBots.length ? scoreFromPercent(pct$2(botAccess.length - blockedBots.length, botAccess.length), 20) : 20 : 8,
6725
7807
  title: robotsUnavailable ? "robots.txt crawler-policy evidence is unavailable" : !hasBotAccessEvidence ? "robots.txt crawler-policy evidence was not collected" : blockedBots.length ? "robots.txt blocks selected crawler tokens at the start URL" : "robots.txt allows selected crawler tokens at the start URL",
6726
7808
  plainEnglish: robotsUnavailable ? "The robots.txt request failed, so this report cannot say whether selected crawler tokens are allowed at the start URL." : !hasBotAccessEvidence ? "This crawl does not include per-bot robots.txt policy data for the start URL." : blockedBots.length ? `robots.txt policy blocks ${blockedBots.length} selected crawler ${blockedBots.length === 1 ? "token" : "tokens"} at the start URL.` : "robots.txt policy does not block the selected crawler tokens at the start URL. This does not verify actual or site-wide fetchability.",
6727
7809
  action: robotsUnavailable ? "Restore a stable robots.txt response, then rerun this report before making crawler-policy claims." : !hasBotAccessEvidence ? "Rerun the crawl with top-level per-bot robots.txt evidence before making crawler-policy claims." : blockedBots.length ? "Only block AI crawlers intentionally. If discovery matters, remove accidental Disallow rules for the blocked user agents." : "Keep robots.txt explicit and intentional so humans and agents can see what is allowed.",
@@ -6764,10 +7846,11 @@ function aiReadiness(report) {
6764
7846
  maxScore: 0,
6765
7847
  score: 0,
6766
7848
  title: "llms.txt is optional",
6767
- plainEnglish: report.ai?.llmsTxt?.exists ? "An optional llms.txt file was found. Google says it has no positive or negative Search impact." : "No llms.txt file was found. Google says the file is not needed for Search and does not affect visibility.",
7849
+ plainEnglish: report.ai?.llmsTxt?.exists ? agentDiscovery?.llmsTxt ? `An optional llms.txt file was found and its body was checked. It declared ${agentDiscovery.llmsTxt.links.length} links, ${agentDiscovery.llmsTxt.duplicateLinks.length} duplicates, and ${agentDiscovery.llmsTxt.missingCrawlRoutes.length} links outside the crawled route inventory. Google says llms.txt has no positive or negative Search impact.` : "An optional llms.txt file was found. Its body was not validated in this crawl. Google says it has no positive or negative Search impact." : "No llms.txt file was found. Google says the file is not needed for Search and does not affect visibility.",
6768
7850
  action: "No SEO action is required. Generate one only for a specific agent or service that consumes it.",
6769
7851
  evidence: {
6770
7852
  llmsTxt: report.ai?.llmsTxt,
7853
+ validation: agentDiscovery?.llmsTxt,
6771
7854
  googleSearchImpact: "none",
6772
7855
  guidanceUrl: "https://developers.google.com/search/updates#clarifying-guidance-on-llms-txt-files"
6773
7856
  }
@@ -6790,7 +7873,7 @@ function aiReadiness(report) {
6790
7873
  evaluated: hasIndexablePages,
6791
7874
  score: 0,
6792
7875
  title: "Short opening paragraphs are an unscored observation",
6793
- plainEnglish: hasIndexablePages ? `${pct$3(answerablePages.length, pageCount)}% of indexable pages have at least one 25-word paragraph near the start. This does not establish content quality, citation likelihood, or Google AI eligibility.` : "No indexable 2xx pages were available, so opening-paragraph coverage was not evaluated.",
7876
+ plainEnglish: hasIndexablePages ? `${pct$2(answerablePages.length, pageCount)}% of indexable pages have at least one 25-word paragraph near the start. This does not establish content quality, citation likelihood, or Google AI eligibility.` : "No indexable 2xx pages were available, so opening-paragraph coverage was not evaluated.",
6794
7877
  action: "Write for the reader and the page intent. Do not create artificial answer blocks or chunk content solely for AI Search.",
6795
7878
  urls: sampleUrls$1(pages, (page) => !page.geo?.answerable),
6796
7879
  evidence: {
@@ -6808,7 +7891,7 @@ function aiReadiness(report) {
6808
7891
  evaluated: hasIndexablePages,
6809
7892
  score: 0,
6810
7893
  title: "Semantic HTML coverage observed",
6811
- plainEnglish: hasIndexablePages ? `${pct$3(semanticPages.length, pageCount)}% of indexable pages use semantic structure.` : "No indexable 2xx pages were available, so semantic HTML coverage was not evaluated.",
7894
+ plainEnglish: hasIndexablePages ? `${pct$2(semanticPages.length, pageCount)}% of indexable pages use semantic structure.` : "No indexable 2xx pages were available, so semantic HTML coverage was not evaluated.",
6812
7895
  action: "Use real headings, lists, tables, nav, main, article, and section elements instead of layout-only divs."
6813
7896
  }),
6814
7897
  check({
@@ -6818,7 +7901,7 @@ function aiReadiness(report) {
6818
7901
  evaluated: hasIndexablePages,
6819
7902
  score: 0,
6820
7903
  title: "Title, H1, and description coverage observed",
6821
- plainEnglish: hasIndexablePages ? `${pct$3(titlePages.length, pageCount)}% of indexable pages have title/H1 coverage and ${pct$3(metaPages.length, pageCount)}% have meta descriptions.` : "No indexable 2xx pages were available, so title, H1, and description coverage was not evaluated.",
7904
+ plainEnglish: hasIndexablePages ? `${pct$2(titlePages.length, pageCount)}% of indexable pages have title/H1 coverage and ${pct$2(metaPages.length, pageCount)}% have meta descriptions.` : "No indexable 2xx pages were available, so title, H1, and description coverage was not evaluated.",
6822
7905
  action: "Write a unique title and descriptive H1. Add a useful meta description when a summary would help searchers understand the page."
6823
7906
  }),
6824
7907
  check({
@@ -6828,7 +7911,7 @@ function aiReadiness(report) {
6828
7911
  evaluated: hasIndexablePages,
6829
7912
  score: 0,
6830
7913
  title: "HTML language declaration coverage observed",
6831
- plainEnglish: hasIndexablePages ? `${pct$3(langPages.length, pageCount)}% of indexable pages declare a language.` : "No indexable 2xx pages were available, so language declaration coverage was not evaluated.",
7914
+ plainEnglish: hasIndexablePages ? `${pct$2(langPages.length, pageCount)}% of indexable pages declare a language.` : "No indexable 2xx pages were available, so language declaration coverage was not evaluated.",
6832
7915
  action: "Set the html lang attribute so crawlers and assistive tech understand the page language."
6833
7916
  })
6834
7917
  ]),
@@ -6839,7 +7922,7 @@ function aiReadiness(report) {
6839
7922
  evaluated: hasIndexablePages,
6840
7923
  score: 0,
6841
7924
  title: "Entity-signal coverage observed",
6842
- plainEnglish: hasIndexablePages ? `${pct$3(entityPages.length, pageCount)}% of indexable pages include entity signals such as Organization, Person, Product, sameAs, or social profile links.` : "No indexable 2xx pages were available, so entity-signal coverage was not evaluated.",
7925
+ plainEnglish: hasIndexablePages ? `${pct$2(entityPages.length, pageCount)}% of indexable pages include entity signals such as Organization, Person, Product, sameAs, or social profile links.` : "No indexable 2xx pages were available, so entity-signal coverage was not evaluated.",
6843
7926
  action: "Make the homepage and key pages explicit about the brand, product, people, and official profiles using accurate schema and links.",
6844
7927
  urls: sampleUrls$1(pages, (page) => !entityPages.includes(page))
6845
7928
  }), check({
@@ -6883,12 +7966,52 @@ function aiReadiness(report) {
6883
7966
  section: "technical-ux",
6884
7967
  maxScore: 12,
6885
7968
  evaluated: report.pages.length > 0,
6886
- score: scoreFromPercent(pct$3(pages.length, report.pages.length || 1), 12),
7969
+ score: scoreFromPercent(pct$2(pages.length, report.pages.length || 1), 12),
6887
7970
  title: report.pages.length ? "Crawled-page indexability observed" : "Crawled-page indexability was not evaluated",
6888
- plainEnglish: report.pages.length ? `${pct$3(pages.length, report.pages.length)}% of crawled pages are indexable 2xx pages.` : "No crawled pages were available, so status and indexability were not evaluated.",
7971
+ plainEnglish: report.pages.length ? `${pct$2(pages.length, report.pages.length)}% of crawled pages are indexable 2xx pages.` : "No crawled pages were available, so status and indexability were not evaluated.",
6889
7972
  action: "Fix 4xx/5xx pages, accidental noindex, and blocked internal URLs before expanding AI discovery work."
6890
7973
  })
6891
7974
  ]),
7975
+ section("agent-content", "Agent content observations", [
7976
+ check({
7977
+ id: "markdown-alternates",
7978
+ section: "agent-content",
7979
+ maxScore: 0,
7980
+ score: 0,
7981
+ evaluated: Boolean(agentDiscovery),
7982
+ title: agentDiscovery ? "Markdown alternatives were checked" : "Markdown alternatives were not evaluated",
7983
+ plainEnglish: agentDiscovery ? `${agentDiscovery.markdownAlternates.advertisedPages} of ${agentDiscovery.markdownAlternates.eligibleHtmlPages} successful HTML pages advertised one Markdown alternative. ${agentDiscovery.markdownAlternates.exactByteMatches} explicit responses matched content negotiation byte for byte.` : "This crawl does not include the focused representation checks, so it cannot say whether HTML pages publish stable Markdown alternatives.",
7984
+ action: "Use the focused agent-readiness report when you need route coverage, content negotiation, byte stability, or extraction-quality evidence.",
7985
+ evidence: agentDiscovery ? {
7986
+ eligibleHtmlPages: agentDiscovery.markdownAlternates.eligibleHtmlPages,
7987
+ advertisedPages: agentDiscovery.markdownAlternates.advertisedPages,
7988
+ exactByteMatches: agentDiscovery.markdownAlternates.exactByteMatches,
7989
+ stableResponses: agentDiscovery.markdownAlternates.stableResponses
7990
+ } : void 0
7991
+ }),
7992
+ check({
7993
+ id: "agent-skills-discovery",
7994
+ section: "agent-content",
7995
+ maxScore: 0,
7996
+ score: 0,
7997
+ evaluated: Boolean(agentDiscovery),
7998
+ title: agentDiscovery ? "Agent Skills discovery was checked" : "Agent Skills discovery was not evaluated",
7999
+ plainEnglish: agentDiscovery ? `${agentDiscovery.agentSkills.skills.length} published skill${agentDiscovery.agentSkills.skills.length === 1 ? "" : "s"} were found. ${agentDiscovery.agentSkills.skills.filter((skill) => skill.digestMatches).length} matched their declared SHA-256 digest.` : "This crawl checked common legacy descriptor paths but did not validate the Agent Skills discovery index or its file digests.",
8000
+ action: "Publish Agent Skills only for real reusable capabilities. Generate the index digest from the exact deployed SKILL.md bytes.",
8001
+ evidence: agentDiscovery?.agentSkills
8002
+ }),
8003
+ check({
8004
+ id: "agent-content-profile",
8005
+ section: "agent-content",
8006
+ maxScore: 0,
8007
+ score: 0,
8008
+ evaluated: Boolean(agentDiscovery),
8009
+ title: agentDiscovery ? "The content profile was selected explicitly" : "No focused agent-readiness profile was selected",
8010
+ plainEnglish: agentDiscovery ? "Document access, representations, discovery, identity, and extraction quality were evaluated. API, application, and commerce checks were marked not applicable rather than failed." : "A normal crawl does not assume that a content site also exposes a public API, remote agent endpoint, or checkout.",
8011
+ action: "Run a different profile only when that public capability really exists.",
8012
+ evidence: agentDiscovery?.profileApplicability
8013
+ })
8014
+ ]),
6892
8015
  section("crawl-completeness", "Crawl completeness", [check({
6893
8016
  id: "crawl-depth",
6894
8017
  section: "crawl-completeness",
@@ -6939,117 +8062,6 @@ function aiReadiness(report) {
6939
8062
  };
6940
8063
  }
6941
8064
  //#endregion
6942
- //#region packages/core/src/analyze/crawler/entity-readiness.ts
6943
- function unique$3(values) {
6944
- return [...new Set(values.filter((value) => Boolean(value)))];
6945
- }
6946
- function pct$2(part, total) {
6947
- return total ? Math.round(part / total * 100) : 0;
6948
- }
6949
- function entityReadiness(report) {
6950
- const pages = report.pages.filter((page) => page.indexable && page.status < 400);
6951
- const total = pages.length || 1;
6952
- const schemaTypes = {};
6953
- for (const page of pages) for (const type of page.schemaTypes ?? []) schemaTypes[type] = (schemaTypes[type] ?? 0) + 1;
6954
- const entitySchemaPages = pages.filter((page) => page.schemaTypes?.some((type) => /^(Organization|LocalBusiness|Person|Product|WebSite)$/i.test(type)));
6955
- const sameAsEvidence = pages.flatMap((page) => page.schemaSameAsEvidence ?? []).filter((evidence) => evidence.subjectTypes.some((type) => /^(Organization|LocalBusiness|Person)$/i.test(type)));
6956
- const siteSameAsEvidence = sameAsEvidence.filter((evidence) => evidence.subjectTypes.some((type) => /^(Organization|LocalBusiness)$/i.test(type)));
6957
- const sameAs = unique$3(sameAsEvidence.map((evidence) => evidence.url));
6958
- const sameAsByType = {};
6959
- for (const evidence of sameAsEvidence) for (const type of evidence.subjectTypes) {
6960
- if (!/^(Organization|LocalBusiness|Person)$/i.test(type)) continue;
6961
- sameAsByType[type] = unique$3([...sameAsByType[type] ?? [], evidence.url]);
6962
- }
6963
- const socialProfiles = unique$3(pages.flatMap((page) => page.socialProfileLinks ?? []));
6964
- const authors = unique$3(pages.map((page) => page.author));
6965
- const authorPages = pages.filter((page) => page.author || page.geo?.hasAuthor);
6966
- const datePages = pages.filter((page) => page.hasDate || page.geo?.hasDate);
6967
- const titlePages = pages.filter((page) => page.title && page.h1);
6968
- const checks = [
6969
- {
6970
- coverage: pct$2(entitySchemaPages.length, total),
6971
- check: {
6972
- id: "entity-schema",
6973
- title: "Entity schema is present",
6974
- plainEnglish: `${pct$2(entitySchemaPages.length, total)}% of evaluated indexable pages include Organization, LocalBusiness, Person, Product, or WebSite schema.`,
6975
- action: "Add accurate entity schema to the homepage and key product, author, organization, and local pages.",
6976
- evidence: { schemaTypes },
6977
- urls: pages.filter((page) => !entitySchemaPages.includes(page)).slice(0, 10).map((page) => page.finalUrl)
6978
- }
6979
- },
6980
- {
6981
- coverage: siteSameAsEvidence.length ? 100 : 0,
6982
- check: {
6983
- id: "same-as",
6984
- title: "Site entity profiles are connected",
6985
- plainEnglish: siteSameAsEvidence.length ? "The crawl found sameAs links attached to Organization or LocalBusiness structured data." : sameAs.length ? "The crawl found Person sameAs links, but no Organization or LocalBusiness profile evidence for the site entity." : socialProfiles.length ? "The crawl found social-domain links, but they are not enough to prove official profile ownership." : "The crawl did not find entity-scoped sameAs evidence.",
6986
- action: "Connect official profiles with sameAs on Organization, Person, or LocalBusiness schema. Use only profiles you control.",
6987
- evidence: {
6988
- sameAs,
6989
- siteSameAs: unique$3(siteSameAsEvidence.map((evidence) => evidence.url)),
6990
- sameAsEvidence,
6991
- unclassifiedSocialLinks: socialProfiles
6992
- }
6993
- }
6994
- },
6995
- {
6996
- coverage: Math.round((pct$2(authorPages.length, total) + pct$2(datePages.length, total)) / 2),
6997
- check: {
6998
- id: "authority-freshness",
6999
- title: "Author and date signals observed",
7000
- plainEnglish: `${pct$2(authorPages.length, total)}% of evaluated indexable pages expose an author signal and ${pct$2(datePages.length, total)}% expose a date signal.`,
7001
- action: "For editorial content, identify authors and show accurate published or modified dates when that information is useful to readers.",
7002
- evidence: {
7003
- authorCoverage: pct$2(authorPages.length, total),
7004
- dateCoverage: pct$2(datePages.length, total),
7005
- authors
7006
- }
7007
- }
7008
- },
7009
- {
7010
- coverage: pct$2(titlePages.length, total),
7011
- check: {
7012
- id: "entity-naming",
7013
- title: "Title and H1 coverage observed",
7014
- plainEnglish: `${pct$2(titlePages.length, total)}% of evaluated indexable pages have both a title and H1.`,
7015
- action: "Review pages missing either signal. Keep brand, product, and person names accurate wherever they appear."
7016
- }
7017
- }
7018
- ].map(({ coverage, check }) => ({
7019
- ...check,
7020
- status: "info",
7021
- evidence: {
7022
- ...check.evidence,
7023
- observedCoveragePercent: coverage
7024
- }
7025
- }));
7026
- const dataStatus = report.status === "completed" && pages.length > 0 ? "complete" : "partial";
7027
- return {
7028
- reportId: report.id,
7029
- url: report.config.url,
7030
- generatedAt: report.generatedAt,
7031
- dataStatus,
7032
- evaluatedPages: pages.length,
7033
- crawlPages: report.pages.length,
7034
- assessment: "evidence-only",
7035
- headline: dataStatus === "partial" ? "Entity evidence is incomplete; treat these findings as scoped to the evaluated pages, not the whole site." : "Entity signals are reported as observations, not a ranking, visibility, or machine-understanding score.",
7036
- caveats: [
7037
- ...report.caveats,
7038
- "This report deliberately has no aggregate entity score. Add schema, authors, dates, and sameAs only where they accurately fit the page and a documented use case.",
7039
- ...dataStatus === "partial" ? [`Entity coverage is based on ${pages.length} evaluated indexable page${pages.length === 1 ? "" : "s"} from a ${report.status} crawl.`] : [],
7040
- ...socialProfiles.length && !sameAs.length ? ["Social-domain links are unclassified navigation evidence, not proof that the linked profiles are official."] : []
7041
- ],
7042
- checks,
7043
- entities: {
7044
- schemaTypes,
7045
- sameAs,
7046
- sameAsByType,
7047
- socialProfiles,
7048
- authors
7049
- }
7050
- };
7051
- }
7052
- //#endregion
7053
8065
  //#region packages/core/src/analyze/crawler/geo-gaps.ts
7054
8066
  const ELIGIBILITY_RULES = /* @__PURE__ */ new Set([
7055
8067
  "connection_error",
@@ -7117,7 +8129,7 @@ function partialReasons(report) {
7117
8129
  if (report.requestEvidenceStatus === "partial") reasons.push("request-evidence-partial");
7118
8130
  if (report.requestEvidenceStatus === "unavailable") reasons.push("request-evidence-unavailable");
7119
8131
  if (report.summary.pageLimitReached) reasons.push("page-limit-reached");
7120
- if (report.summary.skippedUrls > 0) reasons.push("skipped-urls");
8132
+ if (report.summary.skippedUrlsByImpact.coverageAffecting > 0) reasons.push("coverage-affecting-skips");
7121
8133
  if (report.summary.failedRequests > 0) reasons.push("failed-requests");
7122
8134
  if (report.summary.extractionFailures > 0) reasons.push("extraction-failures");
7123
8135
  return reasons;
@@ -7145,6 +8157,8 @@ function geoGapsReport(report, filters = {}) {
7145
8157
  queuedUrls: report.summary.queuedUrls,
7146
8158
  crawledUrls: report.summary.crawledUrls,
7147
8159
  skippedUrls: report.summary.skippedUrls,
8160
+ coverageAffectingSkippedUrls: report.summary.skippedUrlsByImpact.coverageAffecting,
8161
+ nonImpactingSkippedUrls: report.summary.skippedUrlsByImpact.nonImpacting,
7148
8162
  failedRequests: report.summary.failedRequests,
7149
8163
  extractionFailures: report.summary.extractionFailures,
7150
8164
  partialReasons: reasons
@@ -7210,8 +8224,56 @@ function candidatePages(report, options = {}) {
7210
8224
  }
7211
8225
  function auditLlmsTxt(report) {
7212
8226
  const llmsTxt = report.ai?.llmsTxt;
8227
+ const validation = report.agentDiscovery?.llmsTxt;
7213
8228
  const pages = candidatePages(report);
7214
8229
  const issues = [];
8230
+ if (validation?.exists && !/^\s*(?:text\/plain|text\/markdown)\b/iu.test(validation.contentType ?? "")) issues.push({
8231
+ id: "llms-content-type",
8232
+ severity: "medium",
8233
+ title: "llms.txt uses an unexpected content type",
8234
+ plainEnglish: `The file returned ${validation.contentType ?? "no content type"} instead of plain text or Markdown.`,
8235
+ action: "Serve the deterministic text file as text/plain or text/markdown so consumers do not mistake it for HTML.",
8236
+ evidence: { contentType: validation.contentType }
8237
+ });
8238
+ if (validation?.exists && validation.headingCount === 0) issues.push({
8239
+ id: "llms-structure",
8240
+ severity: "medium",
8241
+ title: "llms.txt has no useful heading structure",
8242
+ plainEnglish: "The fetched file did not contain a level-one or level-two Markdown heading.",
8243
+ action: "Start with one clear title, then group a short list of useful entry points under descriptive headings."
8244
+ });
8245
+ if (validation?.duplicateLinks.length) issues.push({
8246
+ id: "llms-duplicate-links",
8247
+ severity: "low",
8248
+ title: "llms.txt repeats the same link",
8249
+ plainEnglish: `${validation.duplicateLinks.length} URL${validation.duplicateLinks.length === 1 ? " appears" : "s appear"} more than once.`,
8250
+ action: "Remove duplicate entries so the file remains short and deliberate.",
8251
+ evidence: { urls: validation.duplicateLinks }
8252
+ });
8253
+ const brokenLinks = validation?.links.filter((link) => !link.status || link.status < 200 || link.status >= 400 || link.error) ?? [];
8254
+ if (brokenLinks.length) issues.push({
8255
+ id: "llms-broken-links",
8256
+ severity: "medium",
8257
+ title: "llms.txt links did not resolve",
8258
+ plainEnglish: `${brokenLinks.length} declared link${brokenLinks.length === 1 ? "" : "s"} did not return a usable response during this audit.`,
8259
+ action: "Update or remove stale entries, then rerun the focused agent-readiness report.",
8260
+ evidence: { links: brokenLinks }
8261
+ });
8262
+ if (validation?.missingCrawlRoutes.length) issues.push({
8263
+ id: "llms-outside-crawl",
8264
+ severity: "low",
8265
+ title: "llms.txt links fall outside the crawl inventory",
8266
+ plainEnglish: `${validation.missingCrawlRoutes.length} same-origin route${validation.missingCrawlRoutes.length === 1 ? " was" : "s were"} not present in the retained crawl.`,
8267
+ action: "Confirm those routes are intentional and indexable, or remove stale declarations.",
8268
+ evidence: { urls: validation.missingCrawlRoutes }
8269
+ });
8270
+ if (validation?.repeatedHashStable === false) issues.push({
8271
+ id: "llms-unstable-body",
8272
+ severity: "medium",
8273
+ title: "llms.txt changed between repeated requests",
8274
+ plainEnglish: "Two requests in the same audit returned different SHA-256 digests.",
8275
+ action: "Generate the file during the build and remove timestamps, random ordering, or runtime rewriting."
8276
+ });
7215
8277
  if (pages.length < 3) issues.push({
7216
8278
  id: "thin-llms-inventory",
7217
8279
  severity: "low",
@@ -7229,7 +8291,7 @@ function auditLlmsTxt(report) {
7229
8291
  optional: true,
7230
8292
  googleSearchImpact: "none",
7231
8293
  guidanceUrl: "https://developers.google.com/search/updates#clarifying-guidance-on-llms-txt-files",
7232
- headline: llmsTxt?.exists ? "An optional llms.txt file is present. It has no Google Search visibility impact." : "No llms.txt file was found. This is not an SEO issue and requires no action.",
8294
+ headline: llmsTxt?.exists ? validation ? issues.length ? `The optional llms.txt file was validated and ${issues.length} content or link issue${issues.length === 1 ? " needs" : "s need"} review. It has no Google Search visibility impact.` : `The optional llms.txt file was validated with ${validation.links.length} resolving link${validation.links.length === 1 ? "" : "s"} and a stable body. It has no Google Search visibility impact.` : "An optional llms.txt file is present, but its body was not validated in this crawl. It has no Google Search visibility impact." : "No llms.txt file was found. This is not an SEO issue and requires no action.",
7233
8295
  issues,
7234
8296
  recommendedPages: pages.slice(0, 25).map((page) => ({
7235
8297
  url: page.finalUrl,
@@ -7329,8 +8391,6 @@ const SCORECARD_CHECK_ORDER = [
7329
8391
  "entity-identity",
7330
8392
  "answerable-content"
7331
8393
  ];
7332
- const ENTITY_SCHEMA_PATTERN = /^(Organization|LocalBusiness|Person|Product|WebSite)$/i;
7333
- const SITE_ENTITY_TYPE_PATTERN = /^(Organization|LocalBusiness)$/i;
7334
8394
  function indexablePages(report) {
7335
8395
  return report.pages.filter((page) => page.indexable && page.status < 400);
7336
8396
  }
@@ -7538,8 +8598,8 @@ function entityIdentityCheck(report) {
7538
8598
  finding: "Entity identity could not be evaluated because no indexable 2xx pages were available.",
7539
8599
  verification
7540
8600
  };
7541
- const hasEntitySchema = Object.keys(entity.entities.schemaTypes).some((type) => ENTITY_SCHEMA_PATTERN.test(type));
7542
- const siteSameAs = Object.entries(entity.entities.sameAsByType).filter(([type]) => SITE_ENTITY_TYPE_PATTERN.test(type)).flatMap(([, urls]) => urls);
8601
+ const hasEntitySchema = Object.keys(entity.entities.schemaTypes).some(isEntityEvidenceSchemaType);
8602
+ const siteSameAs = Object.entries(entity.entities.sameAsByType).filter(([type]) => isSiteIdentitySchemaType(type)).flatMap(([, urls]) => urls);
7543
8603
  const uniqueSiteSameAs = [...new Set(siteSameAs)].sort();
7544
8604
  const status = uniqueSiteSameAs.length ? "pass" : hasEntitySchema ? "warn" : "fail";
7545
8605
  return {
@@ -7552,7 +8612,7 @@ function entityIdentityCheck(report) {
7552
8612
  sameAs: entity.entities.sameAs,
7553
8613
  socialProfiles: entity.entities.socialProfiles
7554
8614
  },
7555
- finding: status === "pass" ? "The crawl found sameAs links attached to Organization or LocalBusiness structured data for the site entity." : status === "warn" ? "Entity schema is present, but no Organization or LocalBusiness sameAs evidence connects the site entity to official profiles." : "No Organization, LocalBusiness, Person, Product, or WebSite schema was found on the evaluated pages.",
8615
+ finding: status === "pass" ? "The crawl found sameAs links attached to Organization or LocalBusiness structured data for the site entity." : status === "warn" ? "Entity schema is present, but no Organization or LocalBusiness sameAs evidence connects the site entity to official profiles." : "No Organization, LocalBusiness, Person, Product, SoftwareApplication, or WebSite schema was found on the evaluated pages.",
7556
8616
  verification
7557
8617
  };
7558
8618
  }
@@ -8045,7 +9105,7 @@ const RULE_DEFINITIONS = [
8045
9105
  whyItMatters: "External links are part of the page experience and source trail. Broken references make content feel stale and less trustworthy.",
8046
9106
  howToFix: "Replace the link with a working source, update the URL, or remove the reference if it no longer supports the page.",
8047
9107
  impactIfIgnored: "Readers and agents following citations hit dead ends, which weakens trust in the content.",
8048
- howToVerify: "Re-run the crawl with external checks enabled and confirm the external link no longer appears in externalLinkChecks as broken."
9108
+ howToVerify: "Re-run the crawl with external checks enabled and confirm the URL no longer has a confirmed-broken externalLinkChecks result."
8049
9109
  },
8050
9110
  {
8051
9111
  id: "orphan_page",
@@ -8706,11 +9766,13 @@ function auditCrawlPages(pages, opts = {}) {
8706
9766
  if (page.hasHsts === false) issues.push(issue("hsts_missing", page, void 0, { header: "strict-transport-security" }));
8707
9767
  }
8708
9768
  for (const link of page.externalLinkChecks ?? []) {
8709
- if (!isBrokenLinkStatus(link.status)) continue;
9769
+ if (link.state !== "confirmed-broken") continue;
8710
9770
  issues.push(issue("broken_external_link", page, link.url, {
8711
9771
  url: link.url,
8712
9772
  status: link.status,
8713
- error: link.error
9773
+ error: link.error,
9774
+ state: link.state,
9775
+ attempts: link.attempts
8714
9776
  }));
8715
9777
  }
8716
9778
  if (page.indexable && (page.internalInlinkCount ?? 0) === 0 && !sameUrl(page.url, opts.startUrl)) issues.push(issue("orphan_page", page, "No internal inlinks", { internalInlinkCount: page.internalInlinkCount ?? 0 }));
@@ -8845,6 +9907,80 @@ function auditCrawlPages(pages, opts = {}) {
8845
9907
  return issues;
8846
9908
  }
8847
9909
  //#endregion
9910
+ //#region packages/core/src/analyze/crawler/crawl-skip-reasons.ts
9911
+ const CRAWL_SKIP_REASONS = [
9912
+ "asset-url",
9913
+ "configured-exclusion",
9914
+ "duplicate-normalization",
9915
+ "invalid-url",
9916
+ "off-origin",
9917
+ "origin-backpressure",
9918
+ "queue-safety-limit",
9919
+ "robots-disallowed",
9920
+ "robots-uncertain",
9921
+ "legacy-unclassified"
9922
+ ];
9923
+ const SKIP_IMPACT = {
9924
+ "asset-url": "non-impacting",
9925
+ "configured-exclusion": "non-impacting",
9926
+ "duplicate-normalization": "non-impacting",
9927
+ "invalid-url": "non-impacting",
9928
+ "off-origin": "non-impacting",
9929
+ "origin-backpressure": "coverage-affecting",
9930
+ "queue-safety-limit": "coverage-affecting",
9931
+ "robots-disallowed": "coverage-affecting",
9932
+ "robots-uncertain": "coverage-affecting",
9933
+ "legacy-unclassified": "coverage-affecting"
9934
+ };
9935
+ const REASON_ORDER = new Map(CRAWL_SKIP_REASONS.map((reason, index) => [reason, index]));
9936
+ function isCrawlSkipReason(value) {
9937
+ return typeof value === "string" && CRAWL_SKIP_REASONS.includes(value);
9938
+ }
9939
+ function validCount$1(value) {
9940
+ return typeof value === "number" && Number.isSafeInteger(value) && value > 0 ? value : 0;
9941
+ }
9942
+ function crawlSkipImpact(reason) {
9943
+ return SKIP_IMPACT[reason];
9944
+ }
9945
+ function normalizeCrawlSkipReasonCounts(input) {
9946
+ const counts = /* @__PURE__ */ new Map();
9947
+ if (Array.isArray(input.skipReasons)) for (const item of input.skipReasons) {
9948
+ if (!item || typeof item !== "object") continue;
9949
+ const { reason, count } = item;
9950
+ if (!isCrawlSkipReason(reason)) continue;
9951
+ const normalizedCount = validCount$1(count);
9952
+ if (!normalizedCount) continue;
9953
+ counts.set(reason, (counts.get(reason) ?? 0) + normalizedCount);
9954
+ }
9955
+ const declaredTotal = validCount$1(input.skippedUrls);
9956
+ const classifiedTotal = [...counts.values()].reduce((total, count) => total + count, 0);
9957
+ if (declaredTotal > classifiedTotal) counts.set("legacy-unclassified", (counts.get("legacy-unclassified") ?? 0) + declaredTotal - classifiedTotal);
9958
+ const skipReasons = [...counts.entries()].map(([reason, count]) => ({
9959
+ reason,
9960
+ impact: crawlSkipImpact(reason),
9961
+ count
9962
+ })).sort((left, right) => (REASON_ORDER.get(left.reason) ?? Number.MAX_SAFE_INTEGER) - (REASON_ORDER.get(right.reason) ?? Number.MAX_SAFE_INTEGER));
9963
+ const skippedUrlsByImpact = skipReasons.reduce((totals, item) => {
9964
+ if (item.impact === "coverage-affecting") totals.coverageAffecting += item.count;
9965
+ else totals.nonImpacting += item.count;
9966
+ return totals;
9967
+ }, {
9968
+ coverageAffecting: 0,
9969
+ nonImpacting: 0
9970
+ });
9971
+ return {
9972
+ skippedUrls: skippedUrlsByImpact.coverageAffecting + skippedUrlsByImpact.nonImpacting,
9973
+ skipReasons,
9974
+ skippedUrlsByImpact
9975
+ };
9976
+ }
9977
+ function crawlSkipReasonCountsFromRecord(input = {}) {
9978
+ return normalizeCrawlSkipReasonCounts({ skipReasons: Object.entries(input).map(([reason, count]) => ({
9979
+ reason,
9980
+ count
9981
+ })) }).skipReasons;
9982
+ }
9983
+ //#endregion
8848
9984
  //#region packages/core/src/analyze/crawler/report.ts
8849
9985
  const SENSITIVE_FIELD_NAMES = /* @__PURE__ */ new Set([
8850
9986
  "authorization",
@@ -8883,6 +10019,7 @@ function normalizeCrawlConfig(input) {
8883
10019
  respectRobots: input.respectRobots ?? true,
8884
10020
  useSitemap: input.useSitemap ?? true,
8885
10021
  checkExternal: input.checkExternal ?? true,
10022
+ checkAgentDiscovery: input.checkAgentDiscovery ?? false,
8886
10023
  js: normalizeJavaScriptRenderingMode(input.js),
8887
10024
  refresh: input.refresh ?? false,
8888
10025
  fetchRate: normalizeFetchRate(input)
@@ -9002,12 +10139,13 @@ function summarizeCrawlReport(input) {
9002
10139
  }
9003
10140
  }
9004
10141
  for (const issue of input.issues) byCategory[issue.category] = (byCategory[issue.category] ?? 0) + 1;
10142
+ const runStats = crawlRunStats(input.pages, input.stats);
9005
10143
  return {
9006
10144
  totalPages: input.pages.length,
9007
10145
  indexablePages: input.pages.filter((page) => page.indexable).length,
9008
10146
  nonIndexablePages: input.pages.filter((page) => !page.indexable).length,
9009
10147
  statusErrors: input.pages.filter((page) => page.status === 0 || page.status >= 400).length,
9010
- ...crawlRunStats(input.pages, input.stats),
10148
+ ...runStats,
9011
10149
  attemptedRequests: input.requestEvidenceStatus === "unavailable" ? 0 : requests.length,
9012
10150
  responseRequests: requests.filter((request) => request.outcome === "response").length,
9013
10151
  failedRequests: failedRequests.length,
@@ -9032,11 +10170,17 @@ function crawlRunStats(pages, stats = {}) {
9032
10170
  for (const url of page.sampleExternalLinks ?? []) discovered.add(url);
9033
10171
  observedInternalLinks += page.outgoingInternalCount;
9034
10172
  }
10173
+ const skips = normalizeCrawlSkipReasonCounts({
10174
+ skippedUrls: stats.skippedUrls,
10175
+ skipReasons: stats.skipReasons ?? crawlSkipReasonCountsFromRecord(stats.skipReasonCounts)
10176
+ });
9035
10177
  return {
9036
10178
  discoveredUrls: stats.discoveredUrls ?? discovered.size,
9037
10179
  queuedUrls: stats.queuedUrls ?? pages.length,
9038
10180
  crawledUrls: stats.crawledUrls ?? pages.length,
9039
- skippedUrls: stats.skippedUrls ?? 0,
10181
+ skippedUrls: skips.skippedUrls,
10182
+ skipReasons: skips.skipReasons,
10183
+ skippedUrlsByImpact: skips.skippedUrlsByImpact,
9040
10184
  failedUrls: stats.failedUrls ?? pages.filter((page) => page.status === 0 || page.status >= 400).length,
9041
10185
  observedInternalLinks: stats.observedInternalLinks ?? observedInternalLinks,
9042
10186
  pageLimitReached: stats.pageLimitReached ?? false
@@ -9126,6 +10270,7 @@ function createCrawlReport(input) {
9126
10270
  ...input.ai ? { ai: sanitizeTenantValue(input.ai) } : {},
9127
10271
  ...input.sitemapDiscovery ? { sitemapDiscovery: sanitizeTenantValue(input.sitemapDiscovery) } : {},
9128
10272
  ...input.externalLinkVerification ? { externalLinkVerification: sanitizeTenantValue(input.externalLinkVerification) } : {},
10273
+ ...input.agentDiscovery ? { agentDiscovery: sanitizeTenantValue(input.agentDiscovery) } : {},
9129
10274
  warnings: sanitizeMessages(input.warnings).sort(compareText$13),
9130
10275
  caveats: sanitizeMessages(input.caveats).sort(compareText$13)
9131
10276
  };
@@ -9160,7 +10305,7 @@ function normalizeLoadedCrawlReport(report) {
9160
10305
  }
9161
10306
  //#endregion
9162
10307
  //#region packages/core/src/analyze/crawler/report-store.ts
9163
- const CRAWL_REPORT_STORAGE_VERSION = 5;
10308
+ const CRAWL_REPORT_STORAGE_VERSION = 6;
9164
10309
  function toMeta(row) {
9165
10310
  return {
9166
10311
  id: row.id,
@@ -9180,7 +10325,7 @@ function storageEnvelope(report) {
9180
10325
  if (report.site) tenant.site = report.site;
9181
10326
  return {
9182
10327
  kind: "seo.crawl_report",
9183
- version: 5,
10328
+ version: 6,
9184
10329
  savedAt: (/* @__PURE__ */ new Date()).toISOString(),
9185
10330
  ...tenant.projectId || tenant.site ? { tenant } : {},
9186
10331
  report
@@ -9216,7 +10361,7 @@ function saveCrawlReportToSqlite(report) {
9216
10361
  totalPages: report.summary.totalPages,
9217
10362
  issueCount: report.issues.length,
9218
10363
  createdAt: report.generatedAt,
9219
- storageVersion: 5
10364
+ storageVersion: 6
9220
10365
  };
9221
10366
  }
9222
10367
  function listCrawlReportsFromSqlite(input = {}) {
@@ -9303,6 +10448,15 @@ function cancellationRace(signal) {
9303
10448
  //#endregion
9304
10449
  //#region packages/core/src/analyze/crawler/external-link-checks.ts
9305
10450
  const EXTERNAL_LINK_SELECTION_LIMIT = 200;
10451
+ const EXTERNAL_LINK_STATES = [
10452
+ "available",
10453
+ "confirmed-broken",
10454
+ "transient",
10455
+ "provider-blocked",
10456
+ "rate-limited",
10457
+ "method-rejected",
10458
+ "unavailable"
10459
+ ];
9306
10460
  function stableHash(value) {
9307
10461
  let hash = 2166136261;
9308
10462
  for (const character of value) hash = Math.imul(hash ^ (character.codePointAt(0) ?? 0), 16777619);
@@ -9372,32 +10526,73 @@ function selectCandidateUrls(candidates, limit) {
9372
10526
  }
9373
10527
  return [...selected];
9374
10528
  }
9375
- async function checkExternalLink(url, timeoutMs, fetch, signal) {
10529
+ function isAvailableStatus(status) {
10530
+ return status !== void 0 && status >= 200 && status < 400;
10531
+ }
10532
+ function isBrokenStatus(status) {
10533
+ return status === 404 || status === 410;
10534
+ }
10535
+ function isMethodRejectedStatus(status) {
10536
+ return status === 405 || status === 501;
10537
+ }
10538
+ function isProviderBlockedStatus(status) {
10539
+ return status === 401 || status === 403 || status === 451;
10540
+ }
10541
+ function isTransientStatus(status) {
10542
+ return status === 408 || status === 425 || status !== void 0 && status >= 500;
10543
+ }
10544
+ function stateForAttempts(attempts) {
10545
+ const last = attempts.at(-1);
10546
+ const statuses = attempts.map((attempt) => attempt.status).filter((status) => status !== void 0);
10547
+ if (isAvailableStatus(last?.status)) return isMethodRejectedStatus(attempts[0]?.status) ? "method-rejected" : attempts.length > 1 ? "transient" : "available";
10548
+ if (attempts.length >= 2 && isBrokenStatus(last?.status) && attempts.at(-2)?.status === last?.status) return "confirmed-broken";
10549
+ if (last?.status === 429) return "rate-limited";
10550
+ if (isProviderBlockedStatus(last?.status)) return "provider-blocked";
10551
+ if (isMethodRejectedStatus(last?.status)) return "method-rejected";
10552
+ if (isTransientStatus(last?.status) || statuses.some(isBrokenStatus) || statuses.length > 0 && attempts.some((attempt) => attempt.error)) return "transient";
10553
+ return "unavailable";
10554
+ }
10555
+ async function requestExternalLink(url, method, timeoutMs, fetch, signal) {
9376
10556
  const controller = abortController({
9377
10557
  timeoutMs,
9378
10558
  signal
9379
10559
  });
9380
10560
  try {
9381
10561
  const response = await fetch(url, {
9382
- method: "HEAD",
10562
+ method,
9383
10563
  redirect: "follow",
9384
10564
  signal: controller.signal
9385
10565
  });
9386
10566
  await response.body?.cancel().catch(() => void 0);
9387
10567
  return {
9388
- url,
10568
+ method,
9389
10569
  status: response.status
9390
10570
  };
9391
10571
  } catch (error) {
9392
10572
  return {
9393
- url,
9394
- status: 0,
10573
+ method,
9395
10574
  error: error instanceof Error ? error.message : String(error)
9396
10575
  };
9397
10576
  } finally {
9398
10577
  controller.cleanup();
9399
10578
  }
9400
10579
  }
10580
+ async function checkExternalLink(url, timeoutMs, fetch, signal) {
10581
+ const attempts = [];
10582
+ attempts.push(await requestExternalLink(url, "HEAD", timeoutMs, fetch, signal));
10583
+ const head = attempts[0];
10584
+ if (!signal?.aborted && !isAvailableStatus(head?.status) && head?.status !== 429 && !isProviderBlockedStatus(head?.status)) attempts.push(await requestExternalLink(url, "GET", timeoutMs, fetch, signal));
10585
+ const last = attempts.at(-1);
10586
+ if (!signal?.aborted && isBrokenStatus(last?.status) && attempts.at(-2)?.status !== last?.status) attempts.push(await requestExternalLink(url, "GET", timeoutMs, fetch, signal));
10587
+ const finalAttempt = attempts.at(-1);
10588
+ return {
10589
+ url,
10590
+ ...finalAttempt?.status !== void 0 ? { status: finalAttempt.status } : {},
10591
+ ...finalAttempt?.error ? { error: finalAttempt.error } : {},
10592
+ state: stateForAttempts(attempts),
10593
+ attempts
10594
+ };
10595
+ }
9401
10596
  async function verifyExternalLinks(input) {
9402
10597
  const { candidates, discoveredLinkOccurrences, retainedLinkOccurrences } = collectCandidates(input.pages);
9403
10598
  const urls = input.signal?.aborted ? [] : selectCandidateUrls(candidates, EXTERNAL_LINK_SELECTION_LIMIT);
@@ -9412,16 +10607,22 @@ async function verifyExternalLinks(input) {
9412
10607
  const pageChecks = (page.sampleExternalLinks ?? []).map((url) => checks.get(url)).filter((value) => Boolean(value));
9413
10608
  if (pageChecks.length) page.externalLinkChecks = pageChecks;
9414
10609
  }
9415
- const fetchedUrls = [...checks.values()].filter((check) => typeof check.status === "number" && check.status > 0).length;
9416
- const failedUrls = checks.size - fetchedUrls;
10610
+ const fetchedUrls = [...checks.values()].filter((check) => check.attempts?.some((attempt) => attempt.status !== void 0)).length;
10611
+ const failedUrls = [...checks.values()].filter((check) => check.state === "unavailable").length;
10612
+ const unresolvedMethodRejectedUrls = [...checks.values()].filter((check) => check.state === "method-rejected" && !isAvailableStatus(check.status)).length;
9417
10613
  const deferredUrls = candidates.length - checks.size;
10614
+ const outcomes = Object.fromEntries(EXTERNAL_LINK_STATES.map((state) => [state, [...checks.values()].filter((check) => check.state === state).length]));
9418
10615
  const warnings = [];
9419
10616
  if (discoveredLinkOccurrences > retainedLinkOccurrences) warnings.push(`Retained ${retainedLinkOccurrences} sampled external link occurrence${retainedLinkOccurrences === 1 ? "" : "s"} from ${discoveredLinkOccurrences} observed occurrence${discoveredLinkOccurrences === 1 ? "" : "s"}.`);
9420
10617
  if (candidates.length > urls.length) warnings.push(`Selected ${urls.length} of ${candidates.length} retained external URLs using the ${EXTERNAL_LINK_SELECTION_LIMIT}-URL verification limit.`);
9421
10618
  if (checks.size < urls.length) warnings.push(`Stopped before checking ${urls.length - checks.size} selected external URL${urls.length - checks.size === 1 ? "" : "s"}.`);
9422
- if (failedUrls) warnings.push(`${failedUrls} external URL request${failedUrls === 1 ? "" : "s"} failed before a response was received.`);
10619
+ if (failedUrls) warnings.push(`${failedUrls} external URL${failedUrls === 1 ? "" : "s"} could not be reached after bounded verification.`);
10620
+ if (outcomes.transient) warnings.push(`${outcomes.transient} external URL${outcomes.transient === 1 ? "" : "s"} returned inconsistent or temporary responses. Recheck ${outcomes.transient === 1 ? "it" : "them"} before changing the source page.`);
10621
+ if (outcomes["provider-blocked"]) warnings.push(`${outcomes["provider-blocked"]} external URL${outcomes["provider-blocked"] === 1 ? "" : "s"} blocked automated verification. Open ${outcomes["provider-blocked"] === 1 ? "it" : "them"} manually before changing the source page.`);
10622
+ if (outcomes["rate-limited"]) warnings.push(`${outcomes["rate-limited"]} external URL${outcomes["rate-limited"] === 1 ? "" : "s"} rate-limited verification. Recheck ${outcomes["rate-limited"] === 1 ? "it" : "them"} later.`);
10623
+ if (unresolvedMethodRejectedUrls) warnings.push(`${unresolvedMethodRejectedUrls} external URL${unresolvedMethodRejectedUrls === 1 ? "" : "s"} rejected both verification methods. Open ${unresolvedMethodRejectedUrls === 1 ? "it" : "them"} manually before changing the source page.`);
9423
10624
  return {
9424
- dataStatus: discoveredLinkOccurrences > retainedLinkOccurrences || deferredUrls > 0 || failedUrls > 0 ? "partial" : "complete",
10625
+ dataStatus: discoveredLinkOccurrences > retainedLinkOccurrences || deferredUrls > 0 || failedUrls > 0 || outcomes.transient > 0 || outcomes["provider-blocked"] > 0 || outcomes["rate-limited"] > 0 || unresolvedMethodRejectedUrls > 0 ? "partial" : "complete",
9425
10626
  discoveredLinkOccurrences,
9426
10627
  retainedUrls: candidates.length,
9427
10628
  selectedUrls: urls.length,
@@ -9429,6 +10630,7 @@ async function verifyExternalLinks(input) {
9429
10630
  failedUrls,
9430
10631
  deferredUrls,
9431
10632
  limit: EXTERNAL_LINK_SELECTION_LIMIT,
10633
+ outcomes,
9432
10634
  warnings
9433
10635
  };
9434
10636
  }
@@ -10029,6 +11231,7 @@ async function crawlOne(url, opts = {}) {
10029
11231
  sampleExternalLinks: uniqueExternalLinks.slice(0, 25),
10030
11232
  internalAnchorSamples: anchorSamples(extracted.links, true),
10031
11233
  externalAnchorSamples: anchorSamples(extracted.links, false),
11234
+ markdownAlternates: extracted.markdownAlternates,
10032
11235
  schemaTypes: extracted.schemaTypes,
10033
11236
  structuredDataFormats: extracted.structuredDataFormats ?? [],
10034
11237
  googleRichResults: richResults.assessments,
@@ -10624,6 +11827,7 @@ function resolveCrawlSiteDependencies(dependencies = {}) {
10624
11827
  queryPagesTopQueriesBatch: injectedMode === "batch" ? dependencies.queryPagesTopQueriesBatch : injectedMode ? void 0 : queryPagesTopQueriesBatch,
10625
11828
  fetchLandingPageValues: dependencies.fetchLandingPageValues ?? fetchLandingPageValues,
10626
11829
  landingValueForUrl: dependencies.landingValueForUrl ?? landingValueForUrl,
11830
+ collectAgentDiscovery: dependencies.collectAgentDiscovery ?? collectAgentDiscovery,
10627
11831
  now: dependencies.now ?? (() => /* @__PURE__ */ new Date())
10628
11832
  };
10629
11833
  }
@@ -10949,6 +12153,11 @@ async function crawlSite(input, dependencies) {
10949
12153
  const followLinks = config.mode === "site";
10950
12154
  let queuedUrls = 0;
10951
12155
  let skippedUrls = 0;
12156
+ const skipReasonCounts = {};
12157
+ const recordSkip = (reason) => {
12158
+ skippedUrls += 1;
12159
+ skipReasonCounts[reason] = (skipReasonCounts[reason] ?? 0) + 1;
12160
+ };
10952
12161
  let robotsDeferredUrls = 0;
10953
12162
  let originBackpressureSkippedUrls = 0;
10954
12163
  let queueSafetySkippedUrls = 0;
@@ -11015,7 +12224,7 @@ async function crawlSite(input, dependencies) {
11015
12224
  const enqueue = (value, depth) => {
11016
12225
  const normalized = normalizeUrl$1(value, config.url);
11017
12226
  if (!normalized) {
11018
- skippedUrls += 1;
12227
+ recordSkip("invalid-url");
11019
12228
  emitStatus("url_skipped", {
11020
12229
  url: value,
11021
12230
  depth,
@@ -11032,7 +12241,7 @@ async function crawlSite(input, dependencies) {
11032
12241
  return;
11033
12242
  }
11034
12243
  if (!sameOrigin(normalized, origin)) {
11035
- skippedUrls += 1;
12244
+ recordSkip("off-origin");
11036
12245
  emitStatus("url_skipped", {
11037
12246
  url: normalized,
11038
12247
  depth,
@@ -11041,7 +12250,7 @@ async function crawlSite(input, dependencies) {
11041
12250
  return;
11042
12251
  }
11043
12252
  if (isLikelyAsset(normalized)) {
11044
- skippedUrls += 1;
12253
+ recordSkip("asset-url");
11045
12254
  emitStatus("url_skipped", {
11046
12255
  url: normalized,
11047
12256
  depth,
@@ -11050,7 +12259,7 @@ async function crawlSite(input, dependencies) {
11050
12259
  return;
11051
12260
  }
11052
12261
  if (!passesFilters(normalized, config.include, config.exclude)) {
11053
- skippedUrls += 1;
12262
+ recordSkip("configured-exclusion");
11054
12263
  emitStatus("url_skipped", {
11055
12264
  url: normalized,
11056
12265
  depth,
@@ -11059,7 +12268,7 @@ async function crawlSite(input, dependencies) {
11059
12268
  return;
11060
12269
  }
11061
12270
  if (queue.length + visited.size + inFlight.size >= config.maxPages * 5) {
11062
- skippedUrls += 1;
12271
+ recordSkip("queue-safety-limit");
11063
12272
  queueSafetySkippedUrls += 1;
11064
12273
  emitStatus("url_skipped", {
11065
12274
  url: normalized,
@@ -11168,7 +12377,7 @@ async function crawlSite(input, dependencies) {
11168
12377
  });
11169
12378
  requests.push(request);
11170
12379
  if (request.outcome === "skipped") {
11171
- skippedUrls += 1;
12380
+ recordSkip(request.reason === "robots-disallowed" ? "robots-disallowed" : request.reason === "robots-deferred" ? "robots-uncertain" : "origin-backpressure");
11172
12381
  if (request.reason === "robots-deferred") {
11173
12382
  robotsDeferredUrls += 1;
11174
12383
  warnings.push(`${task.url}: crawl deferred because robots.txt availability is unknown.`);
@@ -11273,7 +12482,7 @@ async function crawlSite(input, dependencies) {
11273
12482
  page.declaredIndexability = "robots-blocked";
11274
12483
  storeDocument([]);
11275
12484
  warnings.push(`${page.url}: skipped because robots.txt disallows it`);
11276
- skippedUrls += 1;
12485
+ recordSkip("robots-disallowed");
11277
12486
  emitStatus("page_skipped", {
11278
12487
  url: page.url,
11279
12488
  depth: task.depth,
@@ -11360,6 +12569,14 @@ async function crawlSite(input, dependencies) {
11360
12569
  message: "Finished external link checks."
11361
12570
  });
11362
12571
  }
12572
+ const agentDiscovery = !cancelled && config.checkAgentDiscovery ? await deps.collectAgentDiscovery({
12573
+ startUrl: config.url,
12574
+ pages,
12575
+ timeoutMs: config.timeoutMs,
12576
+ fetch: deps.fetch,
12577
+ signal,
12578
+ concurrency: Math.min(config.concurrency, 4)
12579
+ }) : void 0;
11363
12580
  for (const page of pages) {
11364
12581
  if (!page.geo) continue;
11365
12582
  page.geo = {
@@ -11386,6 +12603,7 @@ async function crawlSite(input, dependencies) {
11386
12603
  },
11387
12604
  ...sitemapDiscovery ? { sitemapDiscovery } : {},
11388
12605
  ...externalLinkVerification ? { externalLinkVerification } : {},
12606
+ ...agentDiscovery ? { agentDiscovery } : {},
11389
12607
  status: reportStatus,
11390
12608
  warnings,
11391
12609
  caveats: crawlCaveats({
@@ -11400,6 +12618,7 @@ async function crawlSite(input, dependencies) {
11400
12618
  queuedUrls,
11401
12619
  crawledUrls: pages.length,
11402
12620
  skippedUrls,
12621
+ skipReasonCounts,
11403
12622
  failedUrls,
11404
12623
  observedInternalLinks,
11405
12624
  pageLimitReached
@@ -12884,7 +14103,7 @@ function completenessReasons(report) {
12884
14103
  if (report.status !== "completed") reasons.push(`report-status-${report.status}`);
12885
14104
  if (report.summary.pageLimitReached) reasons.push("crawl-page-limit-reached");
12886
14105
  if (report.requestEvidenceStatus !== "available") reasons.push(`request-evidence-${report.requestEvidenceStatus}`);
12887
- if (report.summary.skippedUrls > 0) reasons.push("urls-skipped");
14106
+ if (report.summary.skippedUrlsByImpact.coverageAffecting > 0) reasons.push("coverage-affecting-urls-skipped");
12888
14107
  if (report.summary.failedRequests > 0) reasons.push("requests-failed");
12889
14108
  if (report.summary.abortedRequests > 0) reasons.push("requests-aborted");
12890
14109
  if (report.summary.extractionFailures > 0) reasons.push("document-extraction-failed");
@@ -12904,6 +14123,8 @@ function snapshotCompleteness(report) {
12904
14123
  requestEvidenceStatus: report.requestEvidenceStatus,
12905
14124
  pageLimitReached: report.summary.pageLimitReached,
12906
14125
  skippedUrls: report.summary.skippedUrls,
14126
+ skipReasons: report.summary.skipReasons,
14127
+ skippedUrlsByImpact: report.summary.skippedUrlsByImpact,
12907
14128
  failedRequests: report.summary.failedRequests,
12908
14129
  abortedRequests: report.summary.abortedRequests,
12909
14130
  extractionFailures: report.summary.extractionFailures,
@@ -13539,6 +14760,169 @@ function explainOkfValidation(validation) {
13539
14760
  };
13540
14761
  }
13541
14762
  //#endregion
14763
+ //#region packages/core/src/analyze/crawler/agent-discovery-schema.ts
14764
+ const agentRepresentationResponseSchema = z.object({
14765
+ requestedUrl: z.string().url(),
14766
+ finalUrl: z.string().url().optional(),
14767
+ status: z.number().int().optional(),
14768
+ contentType: z.string().optional(),
14769
+ bytes: z.number().int().nonnegative().optional(),
14770
+ sha256: z.string().optional(),
14771
+ canonicalUrl: z.string().url().optional(),
14772
+ alternateUrl: z.string().url().optional(),
14773
+ varyAccept: z.boolean(),
14774
+ markdownTokens: z.number().int().nonnegative().optional(),
14775
+ contentSignal: z.string().optional(),
14776
+ error: z.string().optional()
14777
+ });
14778
+ const agentDiscoverySchema = z.object({
14779
+ profile: z.literal("content"),
14780
+ profileApplicability: z.record(z.enum([
14781
+ "content",
14782
+ "api",
14783
+ "application",
14784
+ "commerce"
14785
+ ]), z.object({
14786
+ status: z.enum(["evaluated", "notApplicable"]),
14787
+ reason: z.string()
14788
+ })),
14789
+ dataStatus: z.enum([
14790
+ "complete",
14791
+ "partial",
14792
+ "unavailable"
14793
+ ]),
14794
+ markdownAlternates: z.object({
14795
+ eligibleHtmlPages: z.number().int().nonnegative(),
14796
+ advertisedPages: z.number().int().nonnegative(),
14797
+ evaluatedPages: z.number().int().nonnegative(),
14798
+ exactByteMatches: z.number().int().nonnegative(),
14799
+ stableResponses: z.number().int().nonnegative(),
14800
+ pages: z.array(z.object({
14801
+ htmlUrl: z.string().url(),
14802
+ advertisedUrls: z.array(z.string().url()),
14803
+ htmlAlternateUnique: z.boolean(),
14804
+ httpAlternateUrls: z.array(z.string().url()),
14805
+ explicit: agentRepresentationResponseSchema.optional(),
14806
+ negotiated: agentRepresentationResponseSchema.optional(),
14807
+ repeated: agentRepresentationResponseSchema.optional(),
14808
+ explicitMatchesNegotiated: z.boolean().nullable(),
14809
+ repeatedHashStable: z.boolean().nullable(),
14810
+ markdownCanonicalMatchesHtml: z.boolean().nullable(),
14811
+ quality: z.object({
14812
+ frontmatterTitle: z.boolean(),
14813
+ h1Count: z.number().int().nonnegative(),
14814
+ codeFenceBalanced: z.boolean(),
14815
+ tableRows: z.number().int().nonnegative(),
14816
+ links: z.number().int().nonnegative(),
14817
+ wordCount: z.number().int().nonnegative(),
14818
+ rawHtmlTags: z.number().int().nonnegative(),
14819
+ rawSvgTags: z.number().int().nonnegative(),
14820
+ rawScriptTags: z.number().int().nonnegative(),
14821
+ rawStyleTags: z.number().int().nonnegative(),
14822
+ suspiciousConcatenations: z.number().int().nonnegative(),
14823
+ repeatedLines: z.number().int().nonnegative(),
14824
+ sourceWordCount: z.number().int().nonnegative(),
14825
+ wordRetentionRatio: z.number().nonnegative().nullable(),
14826
+ introductoryCopyRetained: z.boolean().nullable(),
14827
+ navigationOnly: z.boolean()
14828
+ }).optional()
14829
+ }))
14830
+ }),
14831
+ contentNegotiation: z.object({
14832
+ qZeroHonoured: z.boolean().nullable(),
14833
+ qZeroStatus: z.number().int().optional(),
14834
+ qZeroContentType: z.string().optional(),
14835
+ error: z.string().optional()
14836
+ }),
14837
+ routeManifest: z.object({
14838
+ url: z.string().url(),
14839
+ status: z.number().int().optional(),
14840
+ valid: z.boolean(),
14841
+ declaredHtmlRoutes: z.array(z.string()),
14842
+ declaredMarkdownRoutes: z.array(z.string()),
14843
+ missingHtmlRoutes: z.array(z.string()),
14844
+ missingMarkdownRoutes: z.array(z.string()),
14845
+ orphanMarkdownRoutes: z.array(z.string()),
14846
+ error: z.string().optional()
14847
+ }),
14848
+ agentSkills: z.object({
14849
+ indexUrl: z.string().url(),
14850
+ status: z.number().int().optional(),
14851
+ contentType: z.string().optional(),
14852
+ cors: z.string().optional(),
14853
+ validIndex: z.boolean(),
14854
+ skills: z.array(z.object({
14855
+ name: z.string().optional(),
14856
+ url: z.string().url().optional(),
14857
+ status: z.number().int().optional(),
14858
+ contentType: z.string().optional(),
14859
+ declaredDigest: z.string().optional(),
14860
+ observedDigest: z.string().optional(),
14861
+ digestMatches: z.boolean().nullable(),
14862
+ frontmatterValid: z.boolean().nullable(),
14863
+ sameOrigin: z.boolean(),
14864
+ cors: z.string().optional(),
14865
+ error: z.string().optional()
14866
+ })),
14867
+ error: z.string().optional()
14868
+ }),
14869
+ llmsTxt: z.object({
14870
+ url: z.string().url(),
14871
+ exists: z.boolean(),
14872
+ status: z.number().int().optional(),
14873
+ contentType: z.string().optional(),
14874
+ bytes: z.number().int().nonnegative().optional(),
14875
+ sha256: z.string().optional(),
14876
+ repeatedHashStable: z.boolean().nullable(),
14877
+ headingCount: z.number().int().nonnegative(),
14878
+ totalParsedLinks: z.number().int().nonnegative(),
14879
+ linkLimitReached: z.boolean(),
14880
+ links: z.array(z.object({
14881
+ label: z.string(),
14882
+ url: z.string().url(),
14883
+ sameOrigin: z.boolean(),
14884
+ status: z.number().int().optional(),
14885
+ finalUrl: z.string().url().optional(),
14886
+ redirected: z.boolean(),
14887
+ indexableTarget: z.boolean().optional(),
14888
+ error: z.string().optional()
14889
+ })),
14890
+ invalidLinks: z.array(z.string()),
14891
+ duplicateLinks: z.array(z.string().url()),
14892
+ offSiteLinks: z.array(z.string().url()),
14893
+ redirectedLinks: z.array(z.string().url()),
14894
+ nonIndexableLinks: z.array(z.string().url()),
14895
+ missingCrawlRoutes: z.array(z.string().url()),
14896
+ oversized: z.boolean(),
14897
+ error: z.string().optional()
14898
+ }),
14899
+ contentSignals: z.object({
14900
+ htmlValues: z.array(z.string()),
14901
+ markdownValues: z.array(z.string()),
14902
+ missingHtmlPages: z.number().int().nonnegative(),
14903
+ missingMarkdownPages: z.number().int().nonnegative(),
14904
+ consistent: z.boolean().nullable()
14905
+ }),
14906
+ protocolVariants: z.object({
14907
+ http: z.object({
14908
+ url: z.string().url(),
14909
+ status: z.number().int().optional(),
14910
+ location: z.string().url().optional(),
14911
+ permanentRedirectToHttps: z.boolean().nullable(),
14912
+ error: z.string().optional()
14913
+ }),
14914
+ www: z.object({
14915
+ url: z.string().url(),
14916
+ status: z.number().int().optional(),
14917
+ location: z.string().url().optional(),
14918
+ redirectsToPreferredHost: z.boolean().nullable(),
14919
+ error: z.string().optional()
14920
+ }),
14921
+ hstsOnStartPage: z.boolean().nullable()
14922
+ }),
14923
+ warnings: z.array(z.string())
14924
+ });
14925
+ //#endregion
13542
14926
  //#region packages/core/src/analyze/crawler/schemas.ts
13543
14927
  const crawlerRuleSeveritySchema = z.enum([
13544
14928
  "low",
@@ -13963,10 +15347,25 @@ const crawlPageSnapshotSchema = z.object({
13963
15347
  href: z.string().url(),
13964
15348
  text: z.string()
13965
15349
  })).optional(),
15350
+ markdownAlternates: z.array(z.string().url()).optional(),
13966
15351
  externalLinkChecks: z.array(z.object({
13967
15352
  url: z.string().url(),
13968
15353
  status: z.number().int().optional(),
13969
- error: z.string().optional()
15354
+ error: z.string().optional(),
15355
+ state: z.enum([
15356
+ "available",
15357
+ "confirmed-broken",
15358
+ "transient",
15359
+ "provider-blocked",
15360
+ "rate-limited",
15361
+ "method-rejected",
15362
+ "unavailable"
15363
+ ]).optional(),
15364
+ attempts: z.array(z.object({
15365
+ method: z.enum(["HEAD", "GET"]),
15366
+ status: z.number().int().optional(),
15367
+ error: z.string().optional()
15368
+ })).optional()
13970
15369
  })).optional(),
13971
15370
  schemaTypes: z.array(z.string()).optional(),
13972
15371
  structuredDataFormats: z.array(z.enum([
@@ -14247,6 +15646,15 @@ const crawlExternalLinkVerificationSchema = z.object({
14247
15646
  failedUrls: z.number().int().nonnegative(),
14248
15647
  deferredUrls: z.number().int().nonnegative(),
14249
15648
  limit: z.number().int().positive(),
15649
+ outcomes: z.record(z.enum([
15650
+ "available",
15651
+ "confirmed-broken",
15652
+ "transient",
15653
+ "provider-blocked",
15654
+ "rate-limited",
15655
+ "method-rejected",
15656
+ "unavailable"
15657
+ ]), z.number().int().nonnegative()).optional(),
14250
15658
  warnings: z.array(z.string())
14251
15659
  });
14252
15660
  const crawlConfigSchema = z.object({
@@ -14267,6 +15675,7 @@ const crawlConfigSchema = z.object({
14267
15675
  respectRobots: z.boolean(),
14268
15676
  useSitemap: z.boolean(),
14269
15677
  checkExternal: z.boolean(),
15678
+ checkAgentDiscovery: z.boolean(),
14270
15679
  js: z.union([z.enum([
14271
15680
  "auto",
14272
15681
  "on",
@@ -14297,6 +15706,15 @@ const crawlReportSummarySchema = z.object({
14297
15706
  queuedUrls: z.number().int(),
14298
15707
  crawledUrls: z.number().int(),
14299
15708
  skippedUrls: z.number().int(),
15709
+ skipReasons: z.array(z.object({
15710
+ reason: z.enum(CRAWL_SKIP_REASONS),
15711
+ impact: z.enum(["coverage-affecting", "non-impacting"]),
15712
+ count: z.number().int().positive()
15713
+ })),
15714
+ skippedUrlsByImpact: z.object({
15715
+ coverageAffecting: z.number().int().nonnegative(),
15716
+ nonImpacting: z.number().int().nonnegative()
15717
+ }),
14300
15718
  failedUrls: z.number().int(),
14301
15719
  observedInternalLinks: z.number().int(),
14302
15720
  pageLimitReached: z.boolean().default(false),
@@ -14379,6 +15797,7 @@ const crawlReportBaseSchema = z.object({
14379
15797
  ai: crawlAiSignalsSchema.optional(),
14380
15798
  sitemapDiscovery: crawlSitemapDiscoverySchema.optional(),
14381
15799
  externalLinkVerification: crawlExternalLinkVerificationSchema.optional(),
15800
+ agentDiscovery: agentDiscoverySchema.optional(),
14382
15801
  warnings: z.array(z.string()),
14383
15802
  caveats: z.array(z.string())
14384
15803
  });
@@ -20175,7 +21594,7 @@ function crawlInput(input) {
20175
21594
  return {
20176
21595
  pages,
20177
21596
  evidence: sourceEvidence({
20178
- completeness: report ? report.summary.pageLimitReached ? "truncated" : report.status === "completed" ? "complete" : report.status === "partial" ? "partial" : "unknown" : "completeness" in input ? input.completeness : "unknown",
21597
+ completeness: report ? report.summary.pageLimitReached ? "truncated" : report.status === "partial" || report.summary.skippedUrlsByImpact.coverageAffecting > 0 ? "partial" : report.status === "completed" ? "complete" : "unknown" : "completeness" in input ? input.completeness : "unknown",
20179
21598
  rowsReceived: pages.length,
20180
21599
  validUrls: urls.length,
20181
21600
  uniqueUrls: uniqueUrls.size,
@@ -25448,14 +26867,14 @@ async function crawlImplementationQueueWorkflow(input, dependencies) {
25448
26867
  async function diagnosePropertyWorkflow(input) {
25449
26868
  const narrative = await reportNarrative(input);
25450
26869
  const skippedCount = narrative.diagnosis.skippedSections?.length ?? 0;
25451
- const stepSummary = narrative.dataStatus === "complete" ? "Generated diagnosis, movement, change, and monitoring narrative." : skippedCount ? `Generated ${narrative.dataStatus === "unavailable" ? "an" : "a"} ${narrative.dataStatus} diagnosis; ${skippedCount} sections were unavailable.` : "Generated a partial diagnosis because one or more source datasets were incomplete.";
26870
+ const stepSummary = input.skipSearchData ? "Skipped provider-backed search diagnosis because no Search Console property was selected." : narrative.dataStatus === "complete" ? "Generated diagnosis, movement, change, and monitoring narrative." : skippedCount ? `Generated ${narrative.dataStatus === "unavailable" ? "an" : "a"} ${narrative.dataStatus} diagnosis; ${skippedCount} sections were unavailable.` : "Generated a partial diagnosis because one or more source datasets were incomplete.";
25452
26871
  return workflowReport({
25453
26872
  workflow: "diagnose-property",
25454
26873
  site: input.site,
25455
26874
  summary: narrative.headline,
25456
26875
  steps: [{
25457
26876
  tool: "seo_report_narrative",
25458
- status: "completed",
26877
+ status: input.skipSearchData ? "skipped" : "completed",
25459
26878
  summary: stepSummary
25460
26879
  }],
25461
26880
  actions: narrative.priorities.map((priority) => ({
@@ -27192,6 +28611,6 @@ function isoDate(daysAgo = 0) {
27192
28611
  //#region packages/core/src/version.ts
27193
28612
  const SEO_VERSION = version;
27194
28613
  //#endregion
27195
- export { AI_REFERRAL_SOURCES, AI_REFERRAL_SOURCES_VERSION, AI_SEARCH_SCORECARD_METHODOLOGY_ID, AI_SEARCH_SCORECARD_METHODOLOGY_VERSION, AUDIT_PAGE_RULE_IDS, BROWSER_USER_AGENT, CANNIBAL_MAXIMUM_DOMINANT_SHARE, CANNIBAL_MINIMUM_PAGE_SHARE, CRAWL_REPORT_STORAGE_VERSION, CTR_DEFAULT_LIMIT, CTR_DEFAULT_MIN_IMPRESSIONS, CTR_MAX_LIMIT, CTR_MAX_MIN_IMPRESSIONS, DataForSeoProvider, GSC_FINAL_DATA_LAG_DAYS, GSC_TIME_ZONE, INTERNAL_LINK_LEXICAL_TARGET_LIMIT, INTERNAL_LINK_MATCH_EVIDENCE_LIMIT, JsonFileStorageAdapter, LocalCredentialsProvider, OKF_DEFAULT_CONCEPTS, OKF_MAX_CONCEPTS, REPORT_BASELINE_MAX_AGE_MS, REPORT_BASELINE_MAX_DEPTH, REPORT_BASELINE_MAX_PAGES, SCORECARD_CHECK_WEIGHTS, SCORECARD_STATUS_CREDIT, SEO_VERSION, SERP_TITLE_REFERENCE_WIDTH_PX, SERP_TITLE_WIDTH_PROFILE, SemrushProvider, SeoError, ServiceAccountAccessTokenClient, SessionLedger, affectedUrls, aiPromptsForQuery, aiReadiness, aiReferralSourceForValue, aiReferralsReport, aiSearchScorecard, allocateIndexUrlsToProperties, analyzeAiReferralRows, analyzeCannibalRows, analyzeCtrUnderperformersFromRows, analyzeDecay, analyzeIndexCoverageSignals, analyzeInternalLinksFromRows, analyzePageOpportunitiesFromRows, analyzeQueryClustersFromRows, analyzeQuickWinsFromRows, analyzeSecondPageRows, analyzeSegmentRows, analyzeStrikingDistanceRows, analyzeTrafficAnomalyRows, assertUrlMatchesGscProperty, attachCrawlRecommendations, auditCrawlPages, auditCrawlRequests, auditLlmsTxt, auditPage, authStatus, boundedSitemapInventory, buildOkfBundle, buildPseoAuditReportFromRows, cannibalReport, changeMeasurementCsvRows, classifyCommunityIntent, clearCache, clientProfileSchema, clusterPseoTemplates, clusterQueryRows, combineRobotsValues, communityIntentReport, compareCrawlPages, compareCrawlReports, compareSegmentRows, configSchema, contentCoverageRecommendation, contentCoverageType, contentOptimizationFromPageOpportunities, contentOptimizationReport, countLabel, crawlConfigHash, crawlConfigSchema, crawlDefinitionId, crawlDiff, crawlFixToQueueItem, crawlImplementationQueueWorkflow, crawlIssueGroupSchema, crawlIssueSchema, crawlOne, crawlPageSnapshotSchema, crawlReportSchema, crawlReportStore, crawlReportSummarySchema, crawlRequestObservationSchema, crawlRunId, crawlSite, crawlTopFixSchema, crawlerJsonSchemas, crawlerRuleCategorySchema, crawlerRuleIdSchema, crawlerRuleInfoSchema, crawlerRuleRecommendationSchema, crawlerRuleSeveritySchema, crawlerSchemas, createAuthorizedClient, createContentGroup, createCrawlReport, createCtrBenchmarkContext, createGoogleAccessTokenClient, createLogger, createPageRenderer, ctrUnderperformersReport, decayClusterDrafts, decayComparisonRange, decayingReport, deleteChange, deleteClient, deleteContentGroup, deleteCrawlReport, deleteTokens, deriveBrandTerms, detectBrandTerms, diagnoseCsvFiles, diagnoseProperty, diagnosePropertyWorkflow, effectiveRobotsDirectives, effectiveSnippetControl, ensureSeoCliDirs, entityReadiness, estimateSerpTitleWidth, expectedCtrForPosition, explainOkfValidation, explainRule, fetchCruxFieldData, fetchPage, fetchSearchStatusIncidents, fetchSitemapUrls, fileSize, finalGscDateRange, findOverlappingSearchUpdates, formatRelativeExpiry, ga4MatchReason, ga4PropertyIdFromName, ga4ReportQualityWarnings, ga4RequestCanUseCache, ga4RowsToObjects, generateLlmsTxt, geoGaps, geoGapsReport, getAuthModeStatus, getCacheStats, getChange, getClient, getClientConfig, getContentGroup, getDb, getKeywordProvider, getPreviousRun, getPrivacySnapshot, getRepeatedLinkRecoverUrls, getRunPages, getSeoCliPaths, getServiceAccountConfig, getServiceAccountStatus, getTokenStorageStatus, groupCrawlIssues, hasMetaRobotsDirective, hasRule, hasXRobotsDirective, hashKey, indexCoveragePlan, indexCoverageSignals, indexMonitor, indexWatch, inferPseoTemplate, inferTemplateMovement, insertCrawlRun, insertLinkRecoverRun, inspectUrl, internalLinksReport, interpretUpdateCorrelation, isBrandQuery, isCompatibleTechnicalBaseline, isCurrentTechnicalBaseline, isSkippableReportError, isoDate, latestCrawlReport, latestCrawlSummaries, latestFinalGscDate, latestIndexWatchSummary, latestLinkRecoverSummary, linkRecover, linkRecoverRecommendation, listChanges, listClients, listContentGroups, listCrawlReports, listGa4AccountSummaries, listGa4DataStreams, listRules, listSearchUpdates, listSites, loadCrawlReport, localCredentialsProvider, loginWithLoopback, makeRunId, matchGa4WebStreams, measureChange, measureCoverage, metaRobotsDirectives, migrateLegacyMacConfig, missingOAuthClientMessage, missingSharedClientLoginMessage, monitoringStatus, monthlyReport, monthlyReportWorkflow, narrativeCsvFiles, normalizeCrawlConfig, normalizeForCoverage, normalizeHttpUrl, normalizeJavaScriptRenderingMode, normalizeLoadedCrawlReport, okfConceptLimit, pageEvidenceRecommendation, pageOpportunitiesReport, parseOlderThan, parsePseoPath, performanceAudit, performanceReportIsCacheable, planDueIndexUrls, planIndexCoverageFromUrls, plural, propertyMatchesUrl, providerPreferenceSchema, pseoAuditOptions, pseoAuditReport, pseoCsvFiles, pseoIndexStatus, pseoPresentation, publicHttpFetch, queryClusterReport, queryClusterTokens, queryContentCoverageFromPage, queryContentFieldsFromPage, queryFieldCoverage, queryOpportunityRecommendation, queryPageMetrics, queryPageTopQuery, queryPagesMetrics, queryPagesMetricsBatch, queryPagesTopQueries, queryPagesTopQueriesBatch, querySearchAnalytics, quickWinsCsvRows, quickWinsReport, readConfig, readOauthClient, readTokens, recommendCrawlDiffItem, recommendationForRule, recordChange, redirectTrace, refreshAuthToken, refreshPrioritiesCsvFiles, refreshPrioritiesWorkflow, renderCrawlCsv, renderCrawlHtml, renderCrawlMarkdownTickets, renderCrawlPagesCsv, renderCrawlPretty, renderCsv, renderPresentationTablesMarkdown, renderPseoMarkdown, renderWorkflowMarkdown, reportNarrative, reportPresentation, requestHeaders, resolveSegmentRanges, resolveSeoCliPaths, resolveTechnicalBaseline, reviewObservations, roundedPosition, runDoctor, runGa4Report, saveClient, saveCrawlReport, secondPage, segmentComparisonRange, segmentImpact, segmentRangeDays, selectAffectedUrls, selectDueIndexUrls, seoErrorEnvelope, seoToAiQueryReport, sessionSourceInListFilter, setDefaultClient, setTokenStorageMode, shiftIsoDate, shouldExcludeBrandQuery, siteSchema, sqliteCrawlReportStore, strikingDistance, summarizeCrawlReport, pageTechnicalSignals as technicalPageSignals, technicalWatchWorkflow, templateForUrl, toSeoError, tokenSchema, topCrawlRecommendations, topFixes, trafficAnomaly, unavailableSegmentImpactReport, unicodeTokens, updateCorrelation, updatePostmortemCsvFiles, updatePostmortemWorkflow, validateOkfFiles, verifyQueryContent, workflowPresentation, writeConfig, writeOauthClient, writeTokens, xRobotsDirectives };
28614
+ export { AI_REFERRAL_SOURCES, AI_REFERRAL_SOURCES_VERSION, AI_SEARCH_SCORECARD_METHODOLOGY_ID, AI_SEARCH_SCORECARD_METHODOLOGY_VERSION, AUDIT_PAGE_RULE_IDS, BROWSER_USER_AGENT, CANNIBAL_MAXIMUM_DOMINANT_SHARE, CANNIBAL_MINIMUM_PAGE_SHARE, CRAWL_REPORT_STORAGE_VERSION, CRAWL_SKIP_REASONS, CTR_DEFAULT_LIMIT, CTR_DEFAULT_MIN_IMPRESSIONS, CTR_MAX_LIMIT, CTR_MAX_MIN_IMPRESSIONS, DataForSeoProvider, GSC_FINAL_DATA_LAG_DAYS, GSC_TIME_ZONE, INTERNAL_LINK_LEXICAL_TARGET_LIMIT, INTERNAL_LINK_MATCH_EVIDENCE_LIMIT, JsonFileStorageAdapter, LocalCredentialsProvider, OKF_DEFAULT_CONCEPTS, OKF_MAX_CONCEPTS, REPORT_BASELINE_MAX_AGE_MS, REPORT_BASELINE_MAX_DEPTH, REPORT_BASELINE_MAX_PAGES, SCORECARD_CHECK_WEIGHTS, SCORECARD_STATUS_CREDIT, SEO_VERSION, SERP_TITLE_REFERENCE_WIDTH_PX, SERP_TITLE_WIDTH_PROFILE, SemrushProvider, SeoError, ServiceAccountAccessTokenClient, SessionLedger, affectedUrls, agentReadiness, aiPromptsForQuery, aiReadiness, aiReferralSourceForValue, aiReferralsReport, aiSearchScorecard, allocateIndexUrlsToProperties, analyzeAiReferralRows, analyzeCannibalRows, analyzeCtrUnderperformersFromRows, analyzeDecay, analyzeIndexCoverageSignals, analyzeInternalLinksFromRows, analyzePageOpportunitiesFromRows, analyzeQueryClustersFromRows, analyzeQuickWinsFromRows, analyzeSecondPageRows, analyzeSegmentRows, analyzeStrikingDistanceRows, analyzeTrafficAnomalyRows, assertUrlMatchesGscProperty, attachCrawlRecommendations, auditCrawlPages, auditCrawlRequests, auditLlmsTxt, auditPage, authStatus, boundedSitemapInventory, buildOkfBundle, buildPseoAuditReportFromRows, cannibalReport, changeMeasurementCsvRows, classifyCommunityIntent, clearCache, clientProfileSchema, clusterPseoTemplates, clusterQueryRows, collectAgentDiscovery, combineRobotsValues, communityIntentReport, compareCrawlPages, compareCrawlReports, compareSegmentRows, configSchema, contentCoverageRecommendation, contentCoverageType, contentOptimizationFromPageOpportunities, contentOptimizationReport, countLabel, crawlConfigHash, crawlConfigSchema, crawlDefinitionId, crawlDiff, crawlFixToQueueItem, crawlImplementationQueueWorkflow, crawlIssueGroupSchema, crawlIssueSchema, crawlOne, crawlPageSnapshotSchema, crawlReportSchema, crawlReportStore, crawlReportSummarySchema, crawlRequestObservationSchema, crawlRunId, crawlSite, crawlSkipImpact, crawlSkipReasonCountsFromRecord, crawlTopFixSchema, crawlerJsonSchemas, crawlerRuleCategorySchema, crawlerRuleIdSchema, crawlerRuleInfoSchema, crawlerRuleRecommendationSchema, crawlerRuleSeveritySchema, crawlerSchemas, createAuthorizedClient, createContentGroup, createCrawlReport, createCtrBenchmarkContext, createGoogleAccessTokenClient, createLogger, createPageRenderer, ctrUnderperformersReport, decayClusterDrafts, decayComparisonRange, decayingReport, deleteChange, deleteClient, deleteContentGroup, deleteCrawlReport, deleteTokens, deriveBrandTerms, detectBrandTerms, diagnoseCsvFiles, diagnoseProperty, diagnosePropertyWorkflow, effectiveRobotsDirectives, effectiveSnippetControl, ensureSeoCliDirs, entityReadiness, estimateSerpTitleWidth, expectedCtrForPosition, explainOkfValidation, explainRule, fetchCruxFieldData, fetchPage, fetchSearchStatusIncidents, fetchSitemapUrls, fileSize, finalGscDateRange, findOverlappingSearchUpdates, formatRelativeExpiry, ga4MatchReason, ga4PropertyIdFromName, ga4ReportQualityWarnings, ga4RequestCanUseCache, ga4RowsToObjects, generateLlmsTxt, geoGaps, geoGapsReport, getAuthModeStatus, getCacheStats, getChange, getClient, getClientConfig, getContentGroup, getDb, getKeywordProvider, getPreviousRun, getPrivacySnapshot, getRepeatedLinkRecoverUrls, getRunPages, getSeoCliPaths, getServiceAccountConfig, getServiceAccountStatus, getTokenStorageStatus, groupCrawlIssues, hasMetaRobotsDirective, hasRule, hasXRobotsDirective, hashKey, indexCoveragePlan, indexCoverageSignals, indexMonitor, indexWatch, inferPseoTemplate, inferTemplateMovement, insertCrawlRun, insertLinkRecoverRun, inspectUrl, internalLinksReport, interpretUpdateCorrelation, isBrandQuery, isCompatibleTechnicalBaseline, isCurrentTechnicalBaseline, isEntityEvidenceSchemaType, isSiteIdentitySchemaType, isSkippableReportError, isoDate, latestCrawlReport, latestCrawlSummaries, latestFinalGscDate, latestIndexWatchSummary, latestLinkRecoverSummary, linkRecover, linkRecoverRecommendation, listChanges, listClients, listContentGroups, listCrawlReports, listGa4AccountSummaries, listGa4DataStreams, listRules, listSearchUpdates, listSites, loadCrawlReport, localCredentialsProvider, loginWithLoopback, makeRunId, matchGa4WebStreams, measureChange, measureCoverage, metaRobotsDirectives, migrateLegacyMacConfig, missingOAuthClientMessage, missingSharedClientLoginMessage, monitoringStatus, monthlyReport, monthlyReportWorkflow, narrativeCsvFiles, normalizeCrawlConfig, normalizeCrawlSkipReasonCounts, normalizeForCoverage, normalizeHttpUrl, normalizeJavaScriptRenderingMode, normalizeLoadedCrawlReport, okfConceptLimit, pageEvidenceRecommendation, pageOpportunitiesReport, parseOlderThan, parsePseoPath, performanceAudit, performanceReportIsCacheable, planDueIndexUrls, planIndexCoverageFromUrls, plural, propertyMatchesUrl, providerPreferenceSchema, pseoAuditOptions, pseoAuditReport, pseoCsvFiles, pseoIndexStatus, pseoPresentation, publicHttpFetch, queryClusterReport, queryClusterTokens, queryContentCoverageFromPage, queryContentFieldsFromPage, queryFieldCoverage, queryOpportunityRecommendation, queryPageMetrics, queryPageTopQuery, queryPagesMetrics, queryPagesMetricsBatch, queryPagesTopQueries, queryPagesTopQueriesBatch, querySearchAnalytics, quickWinsCsvRows, quickWinsReport, readConfig, readOauthClient, readTokens, recommendCrawlDiffItem, recommendationForRule, recordChange, redirectTrace, refreshAuthToken, refreshPrioritiesCsvFiles, refreshPrioritiesWorkflow, renderCrawlCsv, renderCrawlHtml, renderCrawlMarkdownTickets, renderCrawlPagesCsv, renderCrawlPretty, renderCsv, renderPresentationTablesMarkdown, renderPseoMarkdown, renderWorkflowMarkdown, reportNarrative, reportPresentation, requestHeaders, resolveSegmentRanges, resolveSeoCliPaths, resolveTechnicalBaseline, reviewObservations, roundedPosition, runDoctor, runGa4Report, saveClient, saveCrawlReport, secondPage, segmentComparisonRange, segmentImpact, segmentRangeDays, selectAffectedUrls, selectDueIndexUrls, seoErrorEnvelope, seoToAiQueryReport, sessionSourceInListFilter, setDefaultClient, setTokenStorageMode, shiftIsoDate, shouldExcludeBrandQuery, siteSchema, sqliteCrawlReportStore, strikingDistance, summarizeCrawlReport, pageTechnicalSignals as technicalPageSignals, technicalWatchWorkflow, templateForUrl, toSeoError, tokenSchema, topCrawlRecommendations, topFixes, trafficAnomaly, unavailableSegmentImpactReport, unicodeTokens, updateCorrelation, updatePostmortemCsvFiles, updatePostmortemWorkflow, validateOkfFiles, verifyQueryContent, workflowPresentation, writeConfig, writeOauthClient, writeTokens, xRobotsDirectives };
27196
28615
 
27197
28616
  //# sourceMappingURL=index.js.map