seo-crawl-audit 0.1.0 → 0.1.2

package/README.md

@@ -15,6 +15,7 @@ your computer in readable JSON and self-contained HTML files.
 - a live progress indicator;
 - automatic recovery after an interrupted scan without requesting completed
   pages again;
+- configurable spacing between request starts to reduce load on the site;
 - a partial HTML report that remains useful even if the process is stopped;
 - a saved baseline for later regression checks;
 - exit codes suitable for CI and pull-request checks.
@@ -239,6 +240,7 @@ are worth reviewing but do not fail a normal check.
 --all                   Scan every URL found in the sitemap
 --max-pages <number>    Deprecated alias for --pages
 --concurrency <number>  Concurrent requests (default: 5)
+--delay <ms>            Delay between request starts (default: 100)
 --timeout <ms>          Per-request timeout (default: 10000)
 --sitemap <url>         Use a specific sitemap or sitemap index
 --no-sitemap            Skip sitemap discovery and crawl internal links
@@ -258,9 +260,11 @@ in advance.
 
 ### Baseline JSON
 
-The baseline records the source configuration, sitemap and robots metadata, and
-the extracted SEO state of every scanned page. It is deterministic enough to
-review and commit when a team wants versioned regression protection.
+The baseline records the source configuration, including the request delay,
+sitemap and robots metadata, and the extracted SEO state of every scanned page.
+`check` reuses that delay unless `--delay` overrides it. The file is
+deterministic enough to review and commit when a team wants versioned regression
+protection.
 
 ### Checkpoint NDJSON
 
@@ -320,8 +324,27 @@ when a human-readable CI report is useful.
 ## Responsible crawling
 
 The crawler respects `robots.txt` by default, stays on the starting origin,
-limits concurrency to five requests, and applies a ten-second timeout. Increase
-concurrency carefully and only on sites you are allowed to test.
+starts requests at least 100 milliseconds apart, limits concurrency to five
+in-flight requests, and applies a ten-second timeout. The same request gate is
+used for `robots.txt`, sitemap discovery, sitemap files, and HTML pages.
+
+Use a longer delay for a smaller site or a server you do not control:
+
+```bash
+seo-audit https://example.com/ --delay 500
+```
+
+`--delay 500` allows at most two new request starts per second. The default
+`--delay 100` allows at most ten. A value of `0` disables the delay and is best
+reserved for a local or otherwise trusted environment:
+
+```bash
+seo-audit http://localhost:3000/ --delay 0
+```
+
+A very short value such as `--delay 10` can start up to 100 requests per second,
+so it is not a polite default for a public website. Increase concurrency or
+reduce the delay only on sites you are allowed to test.
 
 ## Current scope
 

package/examples/quotes-toscrape-report.html

@@ -23,6 +23,7 @@
       --shadow: 0 12px 36px rgba(31, 42, 68, 0.08);
     }
     * { box-sizing: border-box; }
+    [hidden] { display: none !important; }
     body {
       margin: 0;
       background: var(--bg);
@@ -160,12 +161,15 @@
       color: #344054;
       font: 12px/1.45 ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace;
     }
-    .empty { display: none; padding: 48px 20px; color: var(--muted); text-align: center; }
+    .empty { padding: 48px 20px; color: var(--muted); text-align: center; }
+    .empty strong { display: block; margin-bottom: 4px; color: var(--text); font-size: 16px; }
+    .empty button { margin-top: 14px; }
     footer { margin-top: 18px; color: var(--muted); text-align: center; }
     @media (max-width: 900px) {
       main { width: min(100% - 20px, 1440px); margin-top: 20px; }
       header { display: block; }
       .cards { grid-template-columns: repeat(2, 1fr); }
+      .card:last-child { grid-column: 1 / -1; }
       .filters { grid-template-columns: 1fr; }
       .results-bar { align-items: flex-start; flex-direction: column; }
     }
@@ -179,7 +183,7 @@
         <h1>SEO baseline audit</h1>
         <div class="meta">https://quotes.toscrape.com/</div>
       </div>
-      <div class="meta">Generated 2026-07-29T22:47:54.589Z</div>
+      <div class="meta">Generated <time id="generated-at" datetime="2026-08-02T15:16:04.770Z">2026-08-02T15:16:04.770Z</time></div>
     </header>
 
     
@@ -226,14 +230,14 @@
         <span><strong>Info</strong> is useful context.</span>
       </div>
       <div class="results-bar">
-        <span id="result-count"></span>
-        <div class="pagination">
+        <span id="result-count" aria-live="polite"></span>
+        <div id="pagination" class="pagination">
           <button id="previous" type="button">Previous</button>
-          <span id="page-info"></span>
+          <span id="page-info" aria-live="polite"></span>
           <button id="next" type="button">Next</button>
         </div>
       </div>
-      <div class="table-wrap">
+      <div id="table-wrap" class="table-wrap">
         <table>
           <thead>
             <tr>
@@ -247,25 +251,43 @@
           <tbody id="issues"></tbody>
         </table>
       </div>
-      <div id="empty" class="empty">No issues match the selected filters.</div>
+      <div id="empty" class="empty" role="status" hidden>
+        <strong id="empty-title"></strong>
+        <span id="empty-message"></span>
+        <button id="clear-filters" type="button">Clear filters</button>
+      </div>
     </section>
     <footer>Generated locally. No report data was uploaded.</footer>
   </main>
 
   <script>
     const report = {"mode":"scan","issues":[{"severity":"warning","rule":"missing-canonical","url":"https://quotes.toscrape.com/","message":"Page does not declare a canonical URL","after":null},{"severity":"warning","rule":"missing-description","url":"https://quotes.toscrape.com/","message":"Page does not have a meta description","after":null},{"severity":"warning","rule":"missing-canonical","url":"https://quotes.toscrape.com/author/Albert-Einstein","message":"Page does not declare a canonical URL","after":null},{"severity":"warning","rule":"missing-description","url":"https://quotes.toscrape.com/author/Albert-Einstein","message":"Page does not have a meta description","after":null},{"severity":"warning","rule":"missing-canonical","url":"https://quotes.toscrape.com/author/J-K-Rowling","message":"Page does not declare a canonical URL","after":null},{"severity":"warning","rule":"missing-description","url":"https://quotes.toscrape.com/author/J-K-Rowling","message":"Page does not have a meta description","after":null},{"severity":"warning","rule":"missing-canonical","url":"https://quotes.toscrape.com/login","message":"Page does not declare a canonical URL","after":null},{"severity":"warning","rule":"missing-description","url":"https://quotes.toscrape.com/login","message":"Page does not have a meta description","after":null},{"severity":"warning","rule":"missing-canonical","url":"https://quotes.toscrape.com/tag/abilities/page/1","message":"Page does not declare a canonical URL","after":null},{"severity":"warning","rule":"missing-description","url":"https://quotes.toscrape.com/tag/abilities/page/1","message":"Page does not have a meta description","after":null},{"severity":"warning","rule":"missing-canonical","url":"https://quotes.toscrape.com/tag/change/page/1","message":"Page does not declare a canonical URL","after":null},{"severity":"warning","rule":"missing-description","url":"https://quotes.toscrape.com/tag/change/page/1","message":"Page does not have a meta description","after":null},{"severity":"warning","rule":"missing-canonical","url":"https://quotes.toscrape.com/tag/choices/page/1","message":"Page does not declare a canonical URL","after":null},{"severity":"warning","rule":"missing-description","url":"https://quotes.toscrape.com/tag/choices/page/1","message":"Page does not have a meta description","after":null},{"severity":"warning","rule":"missing-canonical","url":"https://quotes.toscrape.com/tag/deep-thoughts/page/1","message":"Page does not declare a canonical URL","after":null},{"severity":"warning","rule":"missing-description","url":"https://quotes.toscrape.com/tag/deep-thoughts/page/1","message":"Page does not have a meta description","after":null},{"severity":"warning","rule":"missing-canonical","url":"https://quotes.toscrape.com/tag/thinking/page/1","message":"Page does not declare a canonical URL","after":null},{"severity":"warning","rule":"missing-description","url":"https://quotes.toscrape.com/tag/thinking/page/1","message":"Page does not have a meta description","after":null},{"severity":"warning","rule":"missing-canonical","url":"https://quotes.toscrape.com/tag/world/page/1","message":"Page does not declare a canonical URL","after":null},{"severity":"warning","rule":"missing-description","url":"https://quotes.toscrape.com/tag/world/page/1","message":"Page does not have a meta description","after":null}]};
+    const generatedAt = document.querySelector("#generated-at");
     const search = document.querySelector("#search");
     const severity = document.querySelector("#severity");
     const rule = document.querySelector("#rule");
     const pageSize = document.querySelector("#page-size");
     const tbody = document.querySelector("#issues");
     const empty = document.querySelector("#empty");
+    const emptyTitle = document.querySelector("#empty-title");
+    const emptyMessage = document.querySelector("#empty-message");
+    const clearFilters = document.querySelector("#clear-filters");
+    const tableWrap = document.querySelector("#table-wrap");
     const resultCount = document.querySelector("#result-count");
+    const pagination = document.querySelector("#pagination");
     const previous = document.querySelector("#previous");
     const next = document.querySelector("#next");
     const pageInfo = document.querySelector("#page-info");
     let currentPage = 1;
 
+    const generatedDate = new Date(generatedAt.dateTime);
+    if (!Number.isNaN(generatedDate.getTime())) {
+      generatedAt.textContent = new Intl.DateTimeFormat(undefined, {
+        dateStyle: "medium",
+        timeStyle: "short",
+      }).format(generatedDate);
+    }
+
     const rules = [...new Set(report.issues.map((issue) => issue.rule))].sort();
     for (const value of rules) {
       const option = document.createElement("option");
@@ -354,11 +376,22 @@
         tbody.append(row);
       }
 
-      resultCount.textContent = filtered.length.toLocaleString() + " of " + report.issues.length.toLocaleString() + " issue(s)";
+      const issueLabel = report.issues.length === 1 ? "issue" : "issues";
+      resultCount.textContent = filtered.length.toLocaleString() + " of " + report.issues.length.toLocaleString() + " " + issueLabel;
       pageInfo.textContent = "Page " + currentPage.toLocaleString() + " of " + pageCount.toLocaleString();
       previous.disabled = currentPage === 1;
       next.disabled = currentPage === pageCount;
-      empty.style.display = filtered.length === 0 ? "block" : "none";
+      pagination.hidden = filtered.length === 0 || pageCount === 1;
+      tableWrap.hidden = filtered.length === 0;
+      empty.hidden = filtered.length !== 0;
+      if (filtered.length === 0) {
+        const reportIsClean = report.issues.length === 0;
+        emptyTitle.textContent = reportIsClean ? "No SEO issues found" : "No matching issues";
+        emptyMessage.textContent = reportIsClean
+          ? "The scanned pages passed all current checks."
+          : "Try a different search or clear the selected filters.";
+        clearFilters.hidden = reportIsClean;
+      }
     }
 
     for (const element of [search, severity, rule, pageSize]) {
@@ -377,6 +410,13 @@
       currentPage += 1;
       render();
     });
+    clearFilters.addEventListener("click", () => {
+      search.value = "";
+      severity.value = "";
+      rule.value = "";
+      currentPage = 1;
+      render();
+    });
     render();
   </script>
 </body>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "seo-crawl-audit",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Local-first CLI for SEO audits, resumable crawls, HTML reports, and regression checks.",
   "type": "module",
   "bin": {

package/src/baseline.js

@@ -27,6 +27,7 @@ export function createBaseline(scan) {
     source: {
       startUrl: scan.startUrl,
       maxPages: scan.options.maxPages,
+      requestDelay: scan.options.requestDelay,
       includeQuery: scan.options.includeQuery,
       respectRobots: scan.options.respectRobots,
       sitemap: scan.options.sitemap,

package/src/cli.js

@@ -17,6 +17,7 @@ import { compareBaselines } from "./compare.js";
 import { crawlSite, fetchPages } from "./crawler.js";
 import { writeHtmlReport } from "./html-report.js";
 import { printIssues, summarizeIssues } from "./report.js";
+import { createRequestGate } from "./request-gate.js";
 import {
   discoverSitemapUrl,
   loadSitemapUrls,
@@ -24,7 +25,7 @@ import {
 import { mapUrlToBaseline, mapUrlToTarget } from "./target.js";
 import { normalizeUrl } from "./urls.js";
 
-const VERSION = "0.1.0";
+const VERSION = "0.1.2";
 const DEFAULT_BASELINE = ".seo-audit.json";
 const DEFAULT_REPORT = "seo-audit-report.html";
 
@@ -54,6 +55,7 @@ Options:
   --all                   Scan every URL found in the sitemap
   --max-pages <number>    Deprecated alias for --pages
   --concurrency <number>  Concurrent requests (default: 5)
+  --delay <ms>            Delay between request starts (default: 100)
   --timeout <ms>          Request timeout in milliseconds (default: 10000)
   --sitemap <url>         Seed the crawl from a sitemap or sitemap index
   --no-sitemap            Skip sitemap discovery and crawl internal links
@@ -82,6 +84,18 @@ function positiveInteger(value, name, fallback) {
   return parsed;
 }
 
+function nonNegativeInteger(value, name, fallback) {
+  if (value === undefined) {
+    return fallback;
+  }
+
+  const parsed = Number.parseInt(value, 10);
+  if (!Number.isInteger(parsed) || parsed < 0) {
+    throw new Error(`${name} must be a non-negative integer`);
+  }
+  return parsed;
+}
+
 function parseCliArgs(args) {
   return parseArgs({
     args,
@@ -97,6 +111,7 @@ function parseCliArgs(args) {
       all: { type: "boolean" },
       "max-pages": { type: "string" },
       concurrency: { type: "string" },
+      delay: { type: "string" },
       timeout: { type: "string" },
       sitemap: { type: "string" },
       "no-sitemap": { type: "boolean" },
@@ -147,6 +162,11 @@ function crawlerOptions(values, baseline) {
       baseline?.source.maxPages ?? 100,
     ),
     concurrency: positiveInteger(values.concurrency, "--concurrency", 5),
+    requestDelay: nonNegativeInteger(
+      values.delay,
+      "--delay",
+      baseline?.source.requestDelay ?? 100,
+    ),
     timeout: positiveInteger(values.timeout, "--timeout", 10_000),
     includeQuery:
       values["include-query"] ?? baseline?.source.includeQuery ?? false,
@@ -301,7 +321,8 @@ async function resolveSitemap(startUrl, values, options) {
 
   const discovered = await discoverSitemapUrl(startUrl, {
     timeout: options.timeout,
-    userAgent: "seo-crawl-audit/0.1.0",
+    userAgent: "seo-crawl-audit/0.1.2",
+    requestGate: options.requestGate,
   });
   if (discovered) {
     if (!values.json) {
@@ -327,6 +348,7 @@ async function scanCommand(url, values) {
 
   const output = resolve(values.output ?? DEFAULT_BASELINE);
   const options = crawlerOptions(values);
+  options.requestGate = createRequestGate(options.requestDelay);
   const normalizedStartUrl = normalizeUrl(url, undefined, {
     includeQuery: options.includeQuery,
   });
@@ -632,6 +654,7 @@ async function checkCommand(url, values) {
   const baseline = await readBaseline(baselinePath);
   const startUrl = url ?? baseline.source.startUrl;
   const options = crawlerOptions(values, baseline);
+  options.requestGate = createRequestGate(options.requestDelay);
   if (options.sitemap) {
     options.sitemap = mapUrlToTarget(
       options.sitemap,

package/src/crawler.js

@@ -1,5 +1,6 @@
 import { createHash } from "node:crypto";
 import { extractSeoData } from "./html.js";
+import { createRequestGate } from "./request-gate.js";
 import { isAllowedByRobots, parseRobots } from "./robots.js";
 import { loadSitemapUrls } from "./sitemap.js";
 import {
@@ -8,7 +9,7 @@ import {
   normalizeUrl,
 } from "./urls.js";
 
-const DEFAULT_USER_AGENT = "seo-crawl-audit/0.1.0";
+const DEFAULT_USER_AGENT = "seo-crawl-audit/0.1.2";
 const MAX_HTML_BYTES = 5 * 1024 * 1024;
 const MAX_ROBOTS_BYTES = 512 * 1024;
 
@@ -42,6 +43,7 @@ async function loadRobots(startUrl, options) {
   const robotsUrl = new URL("/robots.txt", startUrl).href;
 
   try {
+    await options.requestGate();
     const response = await fetch(robotsUrl, {
       headers: {
         accept: "text/plain,*/*;q=0.1",
@@ -98,6 +100,7 @@ async function fetchPage(requestUrl, options) {
   }
 
   try {
+    await options.requestGate();
     const response = await fetch(requestUrl, {
       headers: {
         accept: "text/html,application/xhtml+xml;q=0.9,*/*;q=0.1",
@@ -181,6 +184,8 @@ async function fetchPage(requestUrl, options) {
 }
 
 function withDefaults(options = {}) {
+  const requestDelay = options.requestDelay ?? 100;
+
   return {
     maxPages: options.maxPages ?? 100,
     concurrency: options.concurrency ?? 5,
@@ -190,6 +195,9 @@ function withDefaults(options = {}) {
     sitemap: options.sitemap ?? null,
     sitemapData: options.sitemapData ?? null,
     userAgent: options.userAgent ?? DEFAULT_USER_AGENT,
+    requestDelay,
+    requestGate:
+      options.requestGate ?? createRequestGate(requestDelay),
   };
 }
 

package/src/html-report.js

@@ -66,6 +66,7 @@ export function renderHtmlReport({
       --shadow: 0 12px 36px rgba(31, 42, 68, 0.08);
     }
     * { box-sizing: border-box; }
+    [hidden] { display: none !important; }
     body {
       margin: 0;
       background: var(--bg);
@@ -203,12 +204,15 @@ export function renderHtmlReport({
       color: #344054;
       font: 12px/1.45 ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace;
     }
-    .empty { display: none; padding: 48px 20px; color: var(--muted); text-align: center; }
+    .empty { padding: 48px 20px; color: var(--muted); text-align: center; }
+    .empty strong { display: block; margin-bottom: 4px; color: var(--text); font-size: 16px; }
+    .empty button { margin-top: 14px; }
     footer { margin-top: 18px; color: var(--muted); text-align: center; }
     @media (max-width: 900px) {
       main { width: min(100% - 20px, 1440px); margin-top: 20px; }
       header { display: block; }
       .cards { grid-template-columns: repeat(2, 1fr); }
+      .card:last-child { grid-column: 1 / -1; }
       .filters { grid-template-columns: 1fr; }
       .results-bar { align-items: flex-start; flex-direction: column; }
     }
@@ -222,7 +226,7 @@ export function renderHtmlReport({
         <h1>${escapeHtml(reportTitle)}</h1>
         <div class="meta">${escapeHtml(startUrl)}</div>
       </div>
-      <div class="meta">Generated ${escapeHtml(generatedAt)}</div>
+      <div class="meta">Generated <time id="generated-at" datetime="${escapeHtml(generatedAt)}">${escapeHtml(generatedAt)}</time></div>
     </header>
 
     ${partial ? `<div class="notice"><strong>Partial results.</strong> This report contains the pages saved so far. Run the same scan command again to resume without requesting them twice.</div>` : ""}
@@ -269,14 +273,14 @@ export function renderHtmlReport({
         <span><strong>Info</strong> is useful context.</span>
       </div>
       <div class="results-bar">
-        <span id="result-count"></span>
-        <div class="pagination">
+        <span id="result-count" aria-live="polite"></span>
+        <div id="pagination" class="pagination">
           <button id="previous" type="button">Previous</button>
-          <span id="page-info"></span>
+          <span id="page-info" aria-live="polite"></span>
           <button id="next" type="button">Next</button>
         </div>
       </div>
-      <div class="table-wrap">
+      <div id="table-wrap" class="table-wrap">
         <table>
           <thead>
             <tr>
@@ -290,25 +294,43 @@ export function renderHtmlReport({
           <tbody id="issues"></tbody>
         </table>
       </div>
-      <div id="empty" class="empty">No issues match the selected filters.</div>
+      <div id="empty" class="empty" role="status" hidden>
+        <strong id="empty-title"></strong>
+        <span id="empty-message"></span>
+        <button id="clear-filters" type="button">Clear filters</button>
+      </div>
     </section>
     <footer>Generated locally. No report data was uploaded.</footer>
   </main>
 
   <script>
     const report = ${data};
+    const generatedAt = document.querySelector("#generated-at");
     const search = document.querySelector("#search");
     const severity = document.querySelector("#severity");
     const rule = document.querySelector("#rule");
     const pageSize = document.querySelector("#page-size");
     const tbody = document.querySelector("#issues");
     const empty = document.querySelector("#empty");
+    const emptyTitle = document.querySelector("#empty-title");
+    const emptyMessage = document.querySelector("#empty-message");
+    const clearFilters = document.querySelector("#clear-filters");
+    const tableWrap = document.querySelector("#table-wrap");
     const resultCount = document.querySelector("#result-count");
+    const pagination = document.querySelector("#pagination");
     const previous = document.querySelector("#previous");
     const next = document.querySelector("#next");
     const pageInfo = document.querySelector("#page-info");
     let currentPage = 1;
 
+    const generatedDate = new Date(generatedAt.dateTime);
+    if (!Number.isNaN(generatedDate.getTime())) {
+      generatedAt.textContent = new Intl.DateTimeFormat(undefined, {
+        dateStyle: "medium",
+        timeStyle: "short",
+      }).format(generatedDate);
+    }
+
     const rules = [...new Set(report.issues.map((issue) => issue.rule))].sort();
     for (const value of rules) {
       const option = document.createElement("option");
@@ -397,11 +419,22 @@ export function renderHtmlReport({
         tbody.append(row);
       }
 
-      resultCount.textContent = filtered.length.toLocaleString() + " of " + report.issues.length.toLocaleString() + " issue(s)";
+      const issueLabel = report.issues.length === 1 ? "issue" : "issues";
+      resultCount.textContent = filtered.length.toLocaleString() + " of " + report.issues.length.toLocaleString() + " " + issueLabel;
       pageInfo.textContent = "Page " + currentPage.toLocaleString() + " of " + pageCount.toLocaleString();
       previous.disabled = currentPage === 1;
       next.disabled = currentPage === pageCount;
-      empty.style.display = filtered.length === 0 ? "block" : "none";
+      pagination.hidden = filtered.length === 0 || pageCount === 1;
+      tableWrap.hidden = filtered.length === 0;
+      empty.hidden = filtered.length !== 0;
+      if (filtered.length === 0) {
+        const reportIsClean = report.issues.length === 0;
+        emptyTitle.textContent = reportIsClean ? "No SEO issues found" : "No matching issues";
+        emptyMessage.textContent = reportIsClean
+          ? "The scanned pages passed all current checks."
+          : "Try a different search or clear the selected filters.";
+        clearFilters.hidden = reportIsClean;
+      }
     }
 
     for (const element of [search, severity, rule, pageSize]) {
@@ -420,6 +453,13 @@ export function renderHtmlReport({
       currentPage += 1;
       render();
     });
+    clearFilters.addEventListener("click", () => {
+      search.value = "";
+      severity.value = "";
+      rule.value = "";
+      currentPage = 1;
+      render();
+    });
     render();
   </script>
 </body>

package/src/request-gate.js

@@ -0,0 +1,27 @@
+import { setTimeout as sleep } from "node:timers/promises";
+
+export function createRequestGate(
+  intervalMs = 0,
+  { now = Date.now, wait = sleep } = {},
+) {
+  const interval = Math.max(0, intervalMs);
+  let nextStartAt = 0;
+  let queue = Promise.resolve();
+
+  return function waitForRequestSlot() {
+    if (interval === 0) {
+      return Promise.resolve();
+    }
+
+    const slot = queue.then(async () => {
+      const remaining = Math.max(0, nextStartAt - now());
+      if (remaining > 0) {
+        await wait(remaining);
+      }
+      nextStartAt = now() + interval;
+    });
+
+    queue = slot.catch(() => {});
+    return slot;
+  };
+}

package/src/sitemap.js

@@ -30,6 +30,7 @@ function extractLocations(xml) {
 }
 
 async function fetchSitemap(url, options) {
+  await options.requestGate?.();
   const response = await fetch(url, {
     headers: {
       accept: "application/xml,text/xml,text/plain;q=0.9,*/*;q=0.1",
@@ -54,6 +55,7 @@ async function fetchSitemap(url, options) {
 
 async function isSitemapResponse(url, options) {
   try {
+    await options.requestGate?.();
     const response = await fetch(url, {
       method: "GET",
       headers: {
@@ -91,6 +93,7 @@ export async function discoverSitemapUrl(inputUrl, options) {
   const robotsUrl = new URL("/robots.txt", origin).href;
 
   try {
+    await options.requestGate?.();
     const response = await fetch(robotsUrl, {
       headers: {
         accept: "text/plain,*/*;q=0.1",