seo 0.0.15 → 0.2.0

package/evals/seo-to-ai-query.json

@@ -0,0 +1,57 @@
+{
+  "subject": "seo-to-ai-query",
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "Give me a set of prompts to monitor whether AI tools mention us.",
+      "expected_output": "Should use report id seo-to-ai-query to turn real Search Console query wording into a repeatable set of prompts for external AI monitoring. Should follow the discovery flow: seo_list_reports with category ai-search, seo_describe_report with id seo-to-ai-query, then seo_run_report with the required site. Should read dataStatus states empty, filtered, partial, and available, then dateRange, source.possiblyTruncated, filters, selection.invalidRows, and limited rows. Should note methodology.observedAiPromptData and estimatedTrafficLift are both false.",
+      "assertions": [
+        "Uses report id seo-to-ai-query for the monitoring prompt set",
+        "Follows seo_list_reports, seo_describe_report, seo_run_report in order",
+        "Reads dataStatus states empty, filtered, partial, and available",
+        "Reads dateRange, source.possiblyTruncated, filters, and selection.invalidRows",
+        "Notes methodology.observedAiPromptData and estimatedTrafficLift are both false"
+      ],
+      "files": []
+    },
+    {
+      "id": 2,
+      "prompt": "Run the AI monitoring prompt report for sc-domain:example.com over 90 days, top 40 prompts, at least 25 impressions, and exclude brand.",
+      "expected_output": "Should run report id seo-to-ai-query with site sc-domain:example.com, days 90, limit 40, minImpressions 25, and includeBrand false. Should read source.possiblyTruncated and selection.invalidRows before treating the limited rows as complete, then per-item evidenceScope. Should note each prompt inherits the narrow retained-GSC-query scope, and use only parameters defined in the schema.",
+      "assertions": [
+        "Runs report id seo-to-ai-query with days 90, limit 40, and minImpressions 25",
+        "Sets includeBrand false to exclude brand terms",
+        "Reads source.possiblyTruncated and selection.invalidRows before treating rows as complete",
+        "Reads per-item evidenceScope",
+        "Uses only parameters defined in the report schema"
+      ],
+      "files": []
+    },
+    {
+      "id": 3,
+      "prompt": "You generated 40 prompts, so there is clearly AI demand for us and answers are mentioning our brand.",
+      "expected_output": "Should refuse to read the prompt count as demand or mentions. Should state the generated prompt count is not AI demand, and prompts do not prove anyone entered them or that an answer mentioned the site. Should note methodology.observedAiPromptData is false and each prompt inherits the narrow retained-GSC-query scope. Should recommend storing the source query and date range beside later monitoring observations, and point to ai-referrals for measured GA4 referral evidence.",
+      "assertions": [
+        "Refuses to read the prompt count as AI demand or as mentions",
+        "States the prompt count is not AI demand",
+        "States prompts do not prove anyone entered them or that an answer mentioned the site",
+        "Notes each prompt inherits the narrow retained-GSC-query scope",
+        "Points to report id ai-referrals for measured GA4 referral evidence"
+      ],
+      "files": []
+    },
+    {
+      "id": 4,
+      "prompt": "No prompt rows came back, so there are no AI prompt opportunities and this run already covers all our topics.",
+      "expected_output": "Should refuse to read no returned rows as an all-clear or as full topic coverage. Should state no returned rows is not an all-clear when the source is partial or filtered, and check dataStatus, source.possiblyTruncated, filters, and selection.invalidRows. Should note each prompt inherits the narrow retained-GSC-query scope, so the run does not cover every topic, and recommend widening the window or lowering minImpressions before concluding.",
+      "assertions": [
+        "Refuses to read no returned rows as an all-clear",
+        "States no returned rows is not an all-clear when the source is partial or filtered",
+        "Checks dataStatus, source.possiblyTruncated, filters, and selection.invalidRows",
+        "Notes each prompt inherits the narrow retained-GSC-query scope",
+        "Recommends widening the window or lowering minImpressions before concluding"
+      ],
+      "files": []
+    }
+  ]
+}

package/evals/seo.json

@@ -0,0 +1,87 @@
+{
+  "subject": "seo",
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "Something feels off with our Google traffic lately but I can't tell what. Can you dig into it?",
+      "expected_output": "Should recognize this as an open diagnosis with no known cause and route to the main search performance baseline before any focused report. Should follow the compact MCP flow: seo_list_reports to find ids, seo_describe_report once for the chosen id, then seo_run_report with bounded params. Should name search-performance-overview as the right first report and offer the CLI fallback `seo report --project <project> --json` when MCP is unavailable. Should check dataStatus, source completeness, warnings, and caveats before summarizing, and should not invent clicks, sessions, or rankings.",
+      "assertions": [
+        "Routes to search-performance-overview as the first report for an unknown-cause drop",
+        "Uses the seo_list_reports then seo_describe_report then seo_run_report discovery order",
+        "Does not call seo_describe_report for every report up front",
+        "Offers the CLI fallback `seo report --project <project> --json` for the no-MCP case",
+        "Says it will check dataStatus, source completeness, warnings, and caveats before summarizing",
+        "Does not invent clicks, sessions, or ranking numbers"
+      ],
+      "files": []
+    },
+    {
+      "id": 2,
+      "prompt": "Agent task: return a machine-readable diagnosis for sc-domain:example.com. I need the structured result, not a written report.",
+      "expected_output": "Should reason from MCP structuredContent and treat text or Markdown as the display layer only. Should run report id search-performance-overview through seo_run_report for the property. Should check summary.updateAttributionStatus first and treat summary.updateAttribution as update-correlation context rather than the site's overall diagnosis. Should surface dataStatus, selection counts, and caveats in the structured handoff. Should not switch to the narrative Markdown surface when the caller asked for the machine contract.",
+      "assertions": [
+        "Reasons from structuredContent rather than the Markdown or text display layer",
+        "Runs report id search-performance-overview via seo_run_report",
+        "Checks summary.updateAttributionStatus before describing the diagnosis",
+        "Treats summary.updateAttribution as update-correlation context, not the overall verdict",
+        "Preserves dataStatus, selection counts, and caveats in the structured output",
+        "Does not substitute report-narrative or monthly-report Markdown for the machine contract"
+      ],
+      "files": []
+    },
+    {
+      "id": 3,
+      "prompt": "The quick-wins run shows an estimatedCtrClickShortfall of about 2,400. So if we fix those titles we get roughly 2,400 more clicks a month, right?",
+      "expected_output": "Should decline to confirm the number as a forecast. Should explain that quick-win CTR targets and estimatedCtrClickShortfall are deterministic prioritisation heuristics computed against a site-aware CTR benchmark, not estimated lift or promised traffic. Should note that average position is not a stable rank and that a title change does not guarantee CTR movement. Should reframe the shortfall as a way to rank review candidates and recommend measuring a later complete window to see the real effect.",
+      "assertions": [
+        "Refuses to present estimatedCtrClickShortfall as expected or guaranteed clicks",
+        "Labels the CTR target and shortfall as prioritisation heuristics, not lift",
+        "Notes that average position is not a stable daily rank",
+        "Explains the shortfall only ranks review candidates",
+        "Recommends comparing a later complete window to measure the real change",
+        "Does not restate 2,400 clicks as a projected gain"
+      ],
+      "files": []
+    },
+    {
+      "id": 4,
+      "prompt": "Pull the query totals grouped by page for last month so I can report exact monthly click counts per URL to my boss.",
+      "expected_output": "Should warn that grouped GSC query totals can undercount because anonymised query rows are dropped from grouped sums, so the grouped total is not a guaranteed property total. Should keep zero, missing, capped, and complete states distinct and never turn an anonymised or capped subset into a definitive count or zero. Should point the caller at page-dimension totals when a fuller per-URL number is needed and label any grouped figure as a retained subset. Should not present the grouped sum as the exact monthly clicks.",
+      "assertions": [
+        "Warns that grouped GSC query totals can undercount from anonymised rows",
+        "States the grouped sum is a retained subset, not a guaranteed property total",
+        "Keeps capped, missing, and complete states distinct",
+        "Does not present the grouped total as an exact per-URL count",
+        "Suggests the page-dimension total when a fuller number is required"
+      ],
+      "files": []
+    },
+    {
+      "id": 5,
+      "prompt": "I don't care about Search Console right now. I just want the technical issues on the site itself, like broken tags and crawl problems.",
+      "expected_output": "Should route away from the Search Console diagnosis reports and toward technical crawling. Should name report id site-crawl to build a saved baseline and report id top-fixes for a bounded queue, with the CLI path seo crawl --project <project> --max-pages 500 --save --json. Should keep advice structural and diagnostic and not claim search visibility or value from crawl evidence alone. Should offer report id explain-crawl-issue for an unfamiliar rule.",
+      "assertions": [
+        "Routes to technical crawling instead of Search Console diagnosis reports",
+        "Names report id site-crawl for the saved baseline",
+        "Names report id top-fixes for the bounded fix queue",
+        "Shows a seo crawl command with a page cap and --save",
+        "Does not claim search visibility or page value from crawl evidence alone",
+        "Offers report id explain-crawl-issue for unfamiliar rules"
+      ],
+      "files": []
+    },
+    {
+      "id": 6,
+      "prompt": "Just read the report and tell me exactly what position we rank at for 'invoice software' so I can put it in a slide.",
+      "expected_output": "Should explain that GSC average position is an averaged, sampled metric that varies by search, device, country, and day, so it is not a fixed live rank for a single keyword. Should present the observed average position with its window and source rather than a single exact rank, and should flag if the row is capped, filtered, or anonymised. Should offer to confirm current placement with a bounded live check rather than asserting a literal rank from retained data.",
+      "assertions": [
+        "Explains average position is averaged and sampled, not a live rank",
+        "Notes it varies by search, device, country, and day",
+        "Reports the value with its window and source instead of a single exact rank",
+        "Flags capped, filtered, or anonymised rows if present",
+        "Offers a bounded live verification instead of asserting the rank"
+      ],
+      "files": []
+    }
+  ]
+}

package/evals/setup-check.json

@@ -0,0 +1,56 @@
+{
+  "subject": "setup-check",
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "My reports keep erroring out. Is something wrong with my setup?",
+      "expected_output": "Should route to report id setup-check, which checks Google login, scopes, local configuration, and saved defaults before running reports. Should follow the discovery flow: seo_list_reports with category setup, seo_describe_report with id setup-check, then seo_run_report. Should note the report takes no parameters. Should read ok first, then each checks entry status, distinguishing fail from warn, with its detail and fix, then generatedAt.",
+      "assertions": [
+        "Uses report id setup-check to diagnose the local setup",
+        "Discovers or identifies setup-check through the report catalog (MCP tools or seo reports list/describe/run)",
+        "Notes the report takes no parameters",
+        "Reads ok first, then each check status",
+        "Distinguishes fail from warn and reads each detail and fix",
+        "Presents the diagnosis from the actual check results rather than generic setup advice"
+      ],
+      "files": []
+    },
+    {
+      "id": 2,
+      "prompt": "Agent task: run the local setup check and tell me exactly what to fix.",
+      "expected_output": "Should run report id setup-check with no parameters, since the schema is empty, via seo reports run setup-check or seo_run_report. Should read each checks entry, separate fail from warn, and surface the detail and fix for each failed check. Should recommend applying the fix, re-running setup-check, then running a small read-only report for the intended property.",
+      "assertions": [
+        "Runs report id setup-check with no parameters",
+        "Separates fail from warn across the checks",
+        "Surfaces the detail and fix for each failed check",
+        "Recommends applying the fix and re-running setup-check",
+        "Recommends a small read-only report after setup passes"
+      ],
+      "files": []
+    },
+    {
+      "id": 3,
+      "prompt": "Setup check passed, so my token works, I can pull data for any property, and the site's SEO is fine.",
+      "expected_output": "Should refuse the claims. A passing result means local prerequisites look present, not that a token is still accepted by Google or that the account can access a property. A setup pass is not evidence about rankings, indexing, or site health. Should recommend running a small read-only report for the intended property to confirm access.",
+      "assertions": [
+        "Refuses to read a pass as a working token or property access",
+        "States a pass means local prerequisites look present, not accepted by Google",
+        "States a setup pass is not evidence about rankings, indexing, or site health",
+        "Recommends a small read-only report to confirm property access"
+      ],
+      "files": []
+    },
+    {
+      "id": 4,
+      "prompt": "It warns about missing saved defaults, so that is a blocking failure, right?",
+      "expected_output": "Should not treat the warning as a blocking failure. Missing saved defaults are usually warnings because explicit parameters can replace them. Should distinguish warn from fail and note a report can still run when the site is passed explicitly rather than from a saved default.",
+      "assertions": [
+        "Refuses to treat the missing-defaults warning as a blocking failure",
+        "States missing saved defaults are usually warnings because explicit parameters can replace them",
+        "Distinguishes warn from fail",
+        "Notes a report can run with the site passed explicitly"
+      ],
+      "files": []
+    }
+  ]
+}

package/evals/site-crawl.json

@@ -0,0 +1,44 @@
+{
+  "subject": "site-crawl",
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "Can you crawl our site and find the technical SEO problems across our pages? Want a baseline I can come back to.",
+      "expected_output": "Should use report id site-crawl to crawl linked pages, check technical SEO signals, and save a local baseline with a saved crawl id for follow-up. Should follow the discovery flow: seo_describe_report with id site-crawl, then seo_run_report with the required url and a bounded maxPages. Should read status, requestEvidenceStatus, attempted and fetched counts, failures, retained pages, and pageLimitReached first, then data-source states, then warnings and caveats, with pages, requests, and issues kept opt-in. Should preserve the saved crawl id for later filtering or comparison.",
+      "assertions": [
+        "Uses report id site-crawl to crawl and save a baseline",
+        "Passes the required url and a bounded maxPages",
+        "Reads status, requestEvidenceStatus, attempted, fetched, failures, and pageLimitReached first",
+        "Keeps pages, requests, and issues opt-in rather than dumping them",
+        "Preserves the saved crawl id for later filtering or comparison"
+      ],
+      "files": []
+    },
+    {
+      "id": 2,
+      "prompt": "Crawl https://example.com with maxPages 200, maxDepth 3, and js true, then hand me the highest-priority fixes.",
+      "expected_output": "Should run report id site-crawl with url https://example.com, maxPages 200, maxDepth 3, and js true, via seo reports run site-crawl or seo_run_report. Should read status, attempted and fetched counts, failures, retained pages, and pageLimitReached, then route to report id top-fixes against the saved crawl to rank the queue. Should note that if pageLimitReached is true the crawl was capped, and that a crawl is not an index, traffic source, or guarantee of complete site coverage.",
+      "assertions": [
+        "Runs report id site-crawl with maxPages 200, maxDepth 3, and js true",
+        "Reads status, attempted, fetched, failures, retained pages, and pageLimitReached",
+        "Routes to report id top-fixes against the saved crawl to rank fixes",
+        "States a crawl is not an index, traffic source, or complete coverage",
+        "Uses only parameters defined in the report schema"
+      ],
+      "files": []
+    },
+    {
+      "id": 3,
+      "prompt": "The crawl finished and found no critical issues. That means the whole site is clean and every page is fine, right?",
+      "expected_output": "Should refuse to read a finished crawl as a sitewide all-clear. Should explain that a partial or capped crawl cannot support an all-clear or a definitive zero, and check pageLimitReached, attempted versus fetched counts, and failures before concluding. Should state that if JavaScript rendering was disabled that is a discovery boundary, and recommend preserving the saved crawl id and re-running the same configuration to confirm any change.",
+      "assertions": [
+        "Refuses to read the finished crawl as a sitewide all-clear",
+        "States a partial or capped crawl cannot support an all-clear or a definitive zero",
+        "Checks pageLimitReached, attempted versus fetched counts, and failures",
+        "States a disabled JavaScript rendering is a discovery boundary",
+        "Recommends re-running the same configuration to confirm a change"
+      ],
+      "files": []
+    }
+  ]
+}

package/evals/striking-distance.json

@@ -0,0 +1,85 @@
+{
+  "subject": "striking-distance",
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "Find the keywords we're ranking just below page one for so we know what to push on.",
+      "expected_output": "Should use report id striking-distance to find retained query and page pairs with average positions from 11 through 20. Should follow the discovery flow: seo_list_reports with category diagnosis, seo_describe_report with id striking-distance, then seo_run_report with only described parameters. Should describe the 11 to 20 band as a heuristic boundary and note average position varies by search, device, country, and day, so it does not prove a URL consistently ranks on the literal second page. Should check range, dataStatus, and source before the item list.",
+      "assertions": [
+        "Uses report id striking-distance for the near-page-one queue",
+        "Follows seo_list_reports, seo_describe_report, seo_run_report in order",
+        "Describes the 11 to 20 band as a heuristic boundary",
+        "Notes average position varies by search, device, country, and day",
+        "Checks range, dataStatus, and source before the item list",
+        "Does not claim the URLs consistently sit on the literal second page"
+      ],
+      "files": []
+    },
+    {
+      "id": 2,
+      "prompt": "Run striking-distance for sc-domain:example.com over 28 days, minimum 50 impressions, top 10, brand excluded, and verify the top 5 pages.",
+      "expected_output": "Should run report id striking-distance with params site sc-domain:example.com, days 28, minImpressions 50, limit 10, includeBrand false, verifyContent true, verifyLimit 5, via seo reports run striking-distance --params or seo_run_report. Should inspect the schema before adding brandTerms, JavaScript rendering, or fetch-rate controls. Should read selection to see what was invalid, outside the range, branded, below thresholds, eligible, returned, or limited.",
+      "assertions": [
+        "Runs report id striking-distance with days 28 and minImpressions 50",
+        "Sets limit 10, includeBrand false, verifyContent true, verifyLimit 5",
+        "Inspects the schema before adding brandTerms or JavaScript rendering",
+        "Reads selection for invalid, out-of-range, branded, below-threshold, and returned counts",
+        "Uses only parameters defined in the report schema"
+      ],
+      "files": []
+    },
+    {
+      "id": 3,
+      "prompt": "Every row here is on page 2. Just rewrite all of them and we'll jump to page one. Kick that off.",
+      "expected_output": "Should not rewrite pages solely because they sit inside the position band. Should explain the 11 to 20 range is a heuristic boundary and average position varies, so it does not prove a stable second-page rank or predict how many clicks a change produces. Should recommend auditing a small number of high-impression candidates, fixing verified technical problems first, then comparing intent, title framing, coverage, and internal links against the observed query evidence. Should not promise a jump to page one.",
+      "assertions": [
+        "Refuses to rewrite pages solely for being in the position band",
+        "States the 11 to 20 range is a heuristic boundary, not a stable rank",
+        "Does not promise a jump to page one",
+        "Recommends auditing a small number of high-impression candidates",
+        "Fixes verified technical problems before content rewrites",
+        "Compares intent, title framing, coverage, and internal links against query evidence"
+      ],
+      "files": []
+    },
+    {
+      "id": 4,
+      "prompt": "The report returned 10 rows and nothing for our /guides section. So /guides has no near-page-one queries. Right?",
+      "expected_output": "Should not read the empty /guides result as a definitive zero. Should explain that source.possiblyTruncated and retained-row completeness limit any all-clear, and limit only caps returned rows without increasing coverage. Should keep filtered, out-of-range, capped, and complete states distinct. Should recommend raising limit, widening the window, or checking the source cap before concluding /guides has no candidates.",
+      "assertions": [
+        "Refuses to treat the empty /guides result as a definitive zero",
+        "Cites source.possiblyTruncated and retained-row completeness as limits",
+        "Notes limit caps returned rows and does not increase coverage",
+        "Keeps filtered, out-of-range, capped, and complete states distinct",
+        "Recommends widening scope or checking the cap before concluding"
+      ],
+      "files": []
+    },
+    {
+      "id": 5,
+      "prompt": "Content verification came back failed for two of the candidates. Treat them the same as the verified ones and prioritize a rewrite.",
+      "expected_output": "Should keep unverified or failed pages at lower confidence rather than treating them as verified. Should read the verification counts for requested, attempted, verified, technical, and failed. Should fix verified technical problems first and inspect the failed pages before recommending a content rewrite. Should not equate a failed verification with a confirmed content gap.",
+      "assertions": [
+        "Keeps unverified or failed pages at lower confidence",
+        "Reads verification requested, attempted, verified, technical, and failed counts",
+        "Does not treat failed pages the same as verified ones",
+        "Fixes verified technical problems before content rewrites",
+        "Inspects the failed pages before recommending a rewrite"
+      ],
+      "files": []
+    },
+    {
+      "id": 6,
+      "prompt": "Give me the priority score for each row and tell me how much traffic each one will bring once we optimize it.",
+      "expected_output": "Should present the priority score as a prioritisation heuristic described in methodology, not as expected traffic or ranking lift. Should refuse to convert the score into a traffic forecast. Should explain the score orders review candidates and that observed clicks, impressions, CTR, and average position are evidence, not predictions. Should recommend measuring a later complete window to see actual change.",
+      "assertions": [
+        "Presents the priority score as a prioritisation heuristic from methodology",
+        "Refuses to convert the score into a traffic forecast",
+        "Does not present the score as expected clicks or ranking lift",
+        "Explains the score orders review candidates only",
+        "Recommends comparing a later complete window to measure real change"
+      ],
+      "files": []
+    }
+  ]
+}

package/evals/technical-watch.json

@@ -0,0 +1,57 @@
+{
+  "subject": "technical-watch",
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "Set up a check that catches technical SEO regressions after we deploy.",
+      "expected_output": "Should use report id technical-watch to run crawl and index checks together to catch technical regressions on a schedule. Should follow the discovery flow: seo_list_reports with category workflows, seo_describe_report with id technical-watch, then seo_run_report with the required site and at least one crawl, index, or link recovery input. Should read each workflow step and its nested crawl, index, and recovery output independently. Should clarify a successful run does not prove every URL is crawlable, indexed, unchanged, or issue-free.",
+      "assertions": [
+        "Uses report id technical-watch for the recurring crawl and index checks",
+        "Notes it needs at least one crawl, index, or link recovery input",
+        "Follows seo_list_reports, seo_describe_report, seo_run_report in order",
+        "Reads each workflow step and its nested crawl, index, and recovery output independently",
+        "States a successful run does not prove every URL is crawlable, indexed, unchanged, or issue-free"
+      ],
+      "files": []
+    },
+    {
+      "id": 2,
+      "prompt": "Run technical watch for sc-domain:example.com against our sitemap https://example.com/sitemap.xml, inspect up to 50 URLs, and recover broken search-value links from the last 90 days.",
+      "expected_output": "Should run report id technical-watch with site sc-domain:example.com, sitemaps including https://example.com/sitemap.xml, inspectLimit 50, recoverLinks true, and recoverDays 90. Should read the crawl run ids, limits, failures, and changed items, then the inspection selected, unselected-due, attempted, failed, quota-blocked, deferred, and regression states, then recovery clicks-at-risk as history. Should use only parameters defined in the schema.",
+      "assertions": [
+        "Runs report id technical-watch with the sitemap and inspectLimit 50",
+        "Sets recoverLinks true and recoverDays 90 as requested",
+        "Reads crawl run ids, limits, failures, and changed items",
+        "Reads inspection selected, attempted, failed, quota-blocked, deferred, and regression states",
+        "Reads recovery clicks-at-risk as history"
+      ],
+      "files": []
+    },
+    {
+      "id": 3,
+      "prompt": "The run passed, so every page is crawlable, indexed, and issue-free, and all the sitemap URLs are indexed.",
+      "expected_output": "Should refuse the all-clear. Should state a successful run does not prove every URL is crawlable, indexed, unchanged, or issue-free, that URL Inspection is a provider snapshot and sitemap discovery is not proof of indexing, and that partial crawls, capped inventories, quotas, and failed subqueries must stay visible in any all-clear. Should read the crawl limits and inspection states before making any sitewide claim.",
+      "assertions": [
+        "Refuses the sitewide all-clear",
+        "States a successful run does not prove every URL is crawlable, indexed, unchanged, or issue-free",
+        "States URL Inspection is a provider snapshot and sitemap discovery is not proof of indexing",
+        "States partial crawls, capped inventories, quotas, and failed subqueries must stay visible",
+        "Reads the crawl limits and inspection states before any sitewide claim"
+      ],
+      "files": []
+    },
+    {
+      "id": 4,
+      "prompt": "Some inspections were quota-blocked but the rest passed, so there are no index regressions.",
+      "expected_output": "Should refuse to read quota-blocked inspections as clear. Should state quotas and failed subqueries must stay visible in any all-clear, and read the inspection unselected-due, quota-blocked, deferred, and regression states rather than assuming the unchecked URLs are fine. Should keep checked, quota-blocked, deferred, and regressed states distinct and recommend reproducing new failures, then scheduling another bounded run with a rollback path.",
+      "assertions": [
+        "Refuses to read the quota-blocked inspections as clear",
+        "States quotas and failed subqueries must stay visible in any all-clear",
+        "Reads the inspection unselected-due, quota-blocked, deferred, and regression states",
+        "Keeps checked, quota-blocked, deferred, and regressed states distinct",
+        "Recommends reproducing failures then scheduling another bounded run with a rollback path"
+      ],
+      "files": []
+    }
+  ]
+}

package/evals/top-fixes.json

@@ -0,0 +1,44 @@
+{
+  "subject": "top-fixes",
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "Our crawl came back with hundreds of issues and I have no idea where to start. What should we fix first?",
+      "expected_output": "Should use report id top-fixes to turn crawl findings into a short, ranked queue of technical issues to investigate first. Should follow the discovery flow: seo_describe_report with id top-fixes, then seo_run_report from a saved crawl by reportId or site, with a bounded limit. Should read dataSources, summary, warnings, and caveats first, then per-item rule id, affected count, sample URLs, scoreFactors, and whyThisRanks, then effort, fix guidance, and verification. Should treat rank as a triage order among retained findings, not proof of business impact.",
+      "assertions": [
+        "Uses report id top-fixes to build the ranked triage queue",
+        "Works from a saved crawl by reportId or site with a bounded limit",
+        "Reads dataSources, summary, warnings, and caveats first",
+        "Reads per-item rule id, affected count, sample URLs, scoreFactors, and whyThisRanks",
+        "Treats rank as triage order, not proof of business impact"
+      ],
+      "files": []
+    },
+    {
+      "id": 2,
+      "prompt": "Run top-fixes on saved crawl crawl_example_20260710 for category indexability, limit 10, and give me the reasons each one ranks.",
+      "expected_output": "Should run report id top-fixes with reportId crawl_example_20260710, category indexability, and limit 10, via seo reports run top-fixes or seo_run_report. Should read dataSources, summary, warnings, and caveats, then per-item rule id, affected count, sample URLs, scoreFactors, and whyThisRanks, then effort and verification. Should route to report id affected-urls for the same rule and saved crawl to list every affected URL, and note limit bounds the returned items.",
+      "assertions": [
+        "Runs report id top-fixes with reportId crawl_example_20260710, category indexability, and limit 10",
+        "Reads per-item rule id, affected count, sample URLs, scoreFactors, and whyThisRanks",
+        "Routes to report id affected-urls for the same rule to list every URL",
+        "Notes limit bounds the returned items",
+        "Uses only parameters defined in the report schema"
+      ],
+      "files": []
+    },
+    {
+      "id": 3,
+      "prompt": "The number one item here is our single biggest business problem and this is the globally worst issue on the site. Forecast the traffic we'll gain by fixing it.",
+      "expected_output": "Should refuse all three claims. Should explain that rank means highest among retained eligible findings, not globally highest, because the crawl or category may be capped, and that this is not a forecast of rankings, traffic, or revenue. Should note that a numeric zero is meaningful only when the source is complete and missing provider joins stay unavailable. Should recommend confirming the finding with report id affected-urls before treating it as the top priority.",
+      "assertions": [
+        "Refuses to call the top item the single biggest business problem",
+        "States rank means highest among retained eligible findings, not globally highest",
+        "Notes the crawl or category may be capped",
+        "Refuses to forecast rankings, traffic, or revenue from the queue",
+        "Recommends confirming with report id affected-urls before prioritising"
+      ],
+      "files": []
+    }
+  ]
+}

package/evals/traffic-anomaly.json

@@ -0,0 +1,57 @@
+{
+  "subject": "traffic-anomaly",
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "Did anything weird happen with our search traffic lately?",
+      "expected_output": "Should route to report id traffic-anomaly, which finds recent clicks or impressions that moved beyond the site's normal Search Console pattern. Should follow the discovery flow: seo_list_reports with category diagnosis, seo_describe_report with id traffic-anomaly, then seo_run_report with site. Should read coverage with requested dates, expected versus observed days, missing days, invalid rows, and status first, then the anomalies with metric, baseline and comparison dates, percentChange, zScore, and significanceMethod, then direction and significant. Should treat the output as a movement signal, not a cause.",
+      "assertions": [
+        "Uses report id traffic-anomaly for the movement signal",
+        "Follows seo_list_reports, seo_describe_report, seo_run_report in order",
+        "Reads coverage with expected versus observed days and status first",
+        "Reads anomalies with percentChange, zScore, and significanceMethod",
+        "Reads direction and significant",
+        "Treats the output as a movement signal, not a cause"
+      ],
+      "files": []
+    },
+    {
+      "id": 2,
+      "prompt": "Agent task: run traffic-anomaly for sc-domain:example.com with a 90 day baseline and a 7 day recent window.",
+      "expected_output": "Should run report id traffic-anomaly with site sc-domain:example.com, days 90, and recentDays 7, via seo reports run traffic-anomaly or seo_run_report. Should use only site, days, recentDays, and refresh from the schema. Should check coverage status and missing days before reading whether the movement is significant, and if it is, run report id segment-impact next.",
+      "assertions": [
+        "Runs report id traffic-anomaly with days 90 and recentDays 7",
+        "Uses only parameters defined in the traffic-anomaly schema",
+        "Checks coverage status and missing days before reading significance",
+        "Plans to run report id segment-impact if the movement is significant"
+      ],
+      "files": []
+    },
+    {
+      "id": 3,
+      "prompt": "The spike is significant, so the algorithm update caused it. Put that down as the reason.",
+      "expected_output": "Should refuse. Statistical unusualness does not identify a cause, diagnose a defect, or prove an external event affected the property. Should keep the signal separate from a cause and recommend report id update-correlation for update timing and report id segment-impact for where the movement sits.",
+      "assertions": [
+        "Refuses to name the algorithm update as the cause",
+        "States statistical unusualness does not identify a cause or prove an external event",
+        "Keeps the movement signal separate from a cause",
+        "Recommends report id update-correlation for update timing",
+        "Recommends report id segment-impact for where the movement sits"
+      ],
+      "files": []
+    },
+    {
+      "id": 4,
+      "prompt": "percentChange is null, so there was zero change, and just ignore the missing days in the window.",
+      "expected_output": "Should refuse both readings. A null percentage from a zero baseline differs from zero change, and partial coverage can change the baseline. Should read coverage expected versus observed days, missing days, and status before trusting the signal, and not equate a null percentage with a flat result.",
+      "assertions": [
+        "Refuses to read a null percentChange as zero change",
+        "States a null percentage from a zero baseline differs from zero change",
+        "States partial coverage can change the baseline",
+        "Reads coverage expected versus observed days and missing days",
+        "Does not ignore the missing days in the window"
+      ],
+      "files": []
+    }
+  ]
+}

package/evals/update-correlation.json

@@ -0,0 +1,57 @@
+{
+  "subject": "update-correlation",
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "Our traffic moved around the time of a Google update. Can you line those up?",
+      "expected_output": "Should route to report id update-correlation, which compares search movement dates with official Google ranking update windows and labels overlaps as correlation. Should follow the discovery flow: seo_list_reports with category diagnosis, seo_describe_report with id update-correlation, then seo_run_report with site. Should read the classification with attribution and confidence, returned as not-established and none, first, then anomalies, overlappingUpdates, and confounders, then evidence, caveats, and source.",
+      "assertions": [
+        "Uses report id update-correlation for the timeline",
+        "Follows seo_list_reports, seo_describe_report, seo_run_report in order",
+        "Reads classification with attribution not-established and confidence none first",
+        "Reads anomalies, overlappingUpdates, and confounders",
+        "Reads evidence, caveats, and source",
+        "Presents overlaps as correlation, not causation"
+      ],
+      "files": []
+    },
+    {
+      "id": 2,
+      "prompt": "Agent task: run update-correlation for sc-domain:example.com over 180 days with a 14 day recent window and 3 padding days.",
+      "expected_output": "Should run report id update-correlation with site sc-domain:example.com, days 180, recentDays 14, and paddingDays 3, via seo reports run update-correlation or seo_run_report. Should use only site, days, recentDays, paddingDays, and refresh from the schema. Should confirm the traffic window covers the update dates before reading overlaps.",
+      "assertions": [
+        "Runs report id update-correlation with days 180 and recentDays 14",
+        "Sets paddingDays 3 as requested",
+        "Uses only parameters defined in the update-correlation schema",
+        "Confirms the traffic window covers the update dates before reading overlaps"
+      ],
+      "files": []
+    },
+    {
+      "id": 3,
+      "prompt": "The drop overlaps the March core update and nothing else changed, so the update caused it. Confirm the penalty.",
+      "expected_output": "Should refuse. Overlap cannot show the update caused a gain or loss even with large movement and no other recorded change, and temporal overlap is not a penalty, reward, or confirmed effect. Attribution is returned as not-established. Should recommend testing saved deploys, redirects, and content changes that overlap the window before attributing the movement.",
+      "assertions": [
+        "Refuses to confirm the update caused the drop",
+        "States overlap cannot show causation even with large movement and no other change",
+        "States temporal overlap is not a penalty, reward, or confirmed effect",
+        "Notes attribution is returned as not-established",
+        "Recommends testing deploys, redirects, and content changes in the window first"
+      ],
+      "files": []
+    },
+    {
+      "id": 4,
+      "prompt": "There is an overlap but no significant movement, so mark it an impact. And where there is no overlap, that must be what explains the rest of the drop.",
+      "expected_output": "Should refuse both readings. update-overlap-without-significant-movement is not an impact result, and no-update-overlap does not explain remaining movement. Should keep the classification honest and recommend report id segment-impact and report id traffic-anomaly to investigate the movement rather than forcing the update timeline to explain it.",
+      "assertions": [
+        "Refuses to mark an overlap without significant movement as an impact",
+        "States update-overlap-without-significant-movement is not an impact result",
+        "States no-update-overlap does not explain remaining movement",
+        "Recommends report id segment-impact to investigate the movement",
+        "Recommends report id traffic-anomaly to confirm the movement"
+      ],
+      "files": []
+    }
+  ]
+}

package/evals/update-postmortem.json

@@ -0,0 +1,57 @@
+{
+  "subject": "update-postmortem",
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "The March core update seems to have hit us. What happened?",
+      "expected_output": "Should use report id update-postmortem to find the pages and queries that gained or lost around a confirmed Google ranking update. Should follow the discovery flow: seo_list_reports with category workflows, seo_describe_report with id update-postmortem, then seo_run_report with the required site. Should read the workflow step statuses, finalized date windows, and source completeness first, then failed measurements, retained limits, and caveats, then update, insights, templateMovement, and segments. Should clarify the cause remains not established even when dates overlap.",
+      "assertions": [
+        "Uses report id update-postmortem for the winners and losers review",
+        "Follows seo_list_reports, seo_describe_report, seo_run_report in order",
+        "Reads workflow step statuses, finalized date windows, and source completeness first",
+        "Reads update, insights, templateMovement, and segments",
+        "States the cause remains not established even when dates overlap"
+      ],
+      "files": []
+    },
+    {
+      "id": 2,
+      "prompt": "Run the update postmortem for sc-domain:example.com over 90 days with a 28 day recent window, include the change log, and note that we ran a migration as a known confounder.",
+      "expected_output": "Should run report id update-postmortem with site sc-domain:example.com, days 90, recentDays 28, includeChangeLog true, and knownConfounders including the migration. Should read the finalized date windows and source completeness, then failed measurements, retained limits, and caveats, then templateMovement and segments. Should use only parameters defined in the schema.",
+      "assertions": [
+        "Runs report id update-postmortem with days 90 and recentDays 28",
+        "Sets includeChangeLog true and passes the migration in knownConfounders",
+        "Reads the finalized date windows and source completeness",
+        "Reads failed measurements, retained limits, and caveats",
+        "Reads templateMovement and segments"
+      ],
+      "files": []
+    },
+    {
+      "id": 3,
+      "prompt": "The update clearly caused every one of these changes. Tell me our recovery odds and next month's traffic.",
+      "expected_output": "Should refuse every move. Should state the cause remains not established even when dates overlap, and that the report does not claim update causation, recovery probability, or future traffic. Should recommend inspecting representative URLs from the affected segments and reproducing technical changes before any rollback, rather than asserting causation or a forecast.",
+      "assertions": [
+        "Refuses to attribute every change to the update",
+        "States the cause remains not established even when dates overlap",
+        "Refuses to claim recovery probability or future traffic",
+        "Recommends inspecting representative URLs from the affected segments",
+        "Recommends reproducing technical changes before any rollback"
+      ],
+      "files": []
+    },
+    {
+      "id": 4,
+      "prompt": "This winners and losers list is every affected page, even though our recent period is not final yet.",
+      "expected_output": "Should refuse both readings. Should state retained winners and losers are not a complete inventory when sources are capped or filtered, and that the report should be avoided when the comparison periods are incomplete or contain major unrecorded changes. Should read the finalized date windows, retained limits, failed measurements, and caveats, and recommend waiting for final data before treating the list as complete.",
+      "assertions": [
+        "Refuses to treat the list as every affected page",
+        "States retained winners and losers are not a complete inventory when sources are capped or filtered",
+        "States the report should be avoided when the comparison periods are incomplete",
+        "Reads the finalized date windows, retained limits, failed measurements, and caveats",
+        "Recommends waiting for final data before treating the list as complete"
+      ],
+      "files": []
+    }
+  ]
+}