seo-crawl-audit 0.1.2 → 0.7.2

package/src/compare.js

@@ -1,205 +0,0 @@
-const SEVERITY_ORDER = { error: 0, warning: 1 };
-
-function hasDirective(value, directive) {
-  return new RegExp(`(^|[\\s,])${directive}($|[\\s,])`, "i").test(value ?? "");
-}
-
-function addIssue(issues, severity, rule, url, message, before, after) {
-  issues.push({ severity, rule, url, message, before, after });
-}
-
-export function compareBaselines(baseline, current) {
-  const issues = [];
-  const currentPages = new Map(current.pages.map((page) => [page.url, page]));
-
-  if (
-    baseline.robots.sha256 &&
-    current.robots.sha256 &&
-    baseline.robots.sha256 !== current.robots.sha256
-  ) {
-    addIssue(
-      issues,
-      "warning",
-      "robots-changed",
-      current.robots.url,
-      "robots.txt content changed",
-      baseline.robots.sha256,
-      current.robots.sha256,
-    );
-  }
-
-  for (const before of baseline.pages) {
-    const after = currentPages.get(before.url);
-
-    if (!after) {
-      addIssue(
-        issues,
-        "error",
-        "page-missing",
-        before.url,
-        "page was not checked",
-        before.status,
-        null,
-      );
-      continue;
-    }
-
-    if (after.blockedByRobots && !before.blockedByRobots) {
-      addIssue(
-        issues,
-        "error",
-        "robots-blocked",
-        before.url,
-        "page is now blocked by robots.txt",
-        false,
-        true,
-      );
-      continue;
-    }
-
-    if (after.error && !before.error) {
-      addIssue(
-        issues,
-        "error",
-        "page-unreachable",
-        before.url,
-        `page request failed: ${after.error}`,
-        before.error,
-        after.error,
-      );
-      continue;
-    }
-
-    if (
-      before.status !== null &&
-      before.status < 400 &&
-      (after.status === null || after.status >= 400)
-    ) {
-      addIssue(
-        issues,
-        "error",
-        "status-regression",
-        before.url,
-        `HTTP status regressed from ${before.status} to ${after.status ?? "none"}`,
-        before.status,
-        after.status,
-      );
-    }
-
-    if (
-      !hasDirective(before.robots, "noindex") &&
-      hasDirective(after.robots, "noindex")
-    ) {
-      addIssue(
-        issues,
-        "error",
-        "new-noindex",
-        before.url,
-        "page now contains a noindex directive",
-        before.robots,
-        after.robots,
-      );
-    }
-
-    if (before.title && !after.title) {
-      addIssue(
-        issues,
-        "error",
-        "title-removed",
-        before.url,
-        "page title was removed",
-        before.title,
-        after.title,
-      );
-    } else if (before.title && after.title && before.title !== after.title) {
-      addIssue(
-        issues,
-        "warning",
-        "title-changed",
-        before.url,
-        "page title changed",
-        before.title,
-        after.title,
-      );
-    }
-
-    if (before.description && !after.description) {
-      addIssue(
-        issues,
-        "warning",
-        "description-removed",
-        before.url,
-        "meta description was removed",
-        before.description,
-        after.description,
-      );
-    }
-
-    if (before.canonical && !after.canonical) {
-      addIssue(
-        issues,
-        "error",
-        "canonical-removed",
-        before.url,
-        "canonical URL was removed",
-        before.canonical,
-        after.canonical,
-      );
-    } else if (
-      before.canonical &&
-      after.canonical &&
-      before.canonical !== after.canonical
-    ) {
-      const pageOrigin = new URL(before.url).origin;
-      const canonicalOrigin = new URL(after.canonical).origin;
-      addIssue(
-        issues,
-        canonicalOrigin === pageOrigin ? "warning" : "error",
-        "canonical-changed",
-        before.url,
-        "canonical URL changed",
-        before.canonical,
-        after.canonical,
-      );
-    }
-
-    if ((before.h1Count ?? 0) > 0 && (after.h1Count ?? 0) === 0) {
-      addIssue(
-        issues,
-        "warning",
-        "h1-removed",
-        before.url,
-        "all H1 headings were removed",
-        before.h1Count,
-        after.h1Count,
-      );
-    }
-
-    if (
-      before.finalUrl &&
-      after.finalUrl &&
-      before.finalUrl !== after.finalUrl
-    ) {
-      const pageOrigin = new URL(before.url).origin;
-      const redirectOrigin = new URL(after.finalUrl).origin;
-      addIssue(
-        issues,
-        redirectOrigin === pageOrigin ? "warning" : "error",
-        "redirect-changed",
-        before.url,
-        "final redirect URL changed",
-        before.finalUrl,
-        after.finalUrl,
-      );
-    }
-  }
-
-  issues.sort(
-    (left, right) =>
-      SEVERITY_ORDER[left.severity] - SEVERITY_ORDER[right.severity] ||
-      left.url.localeCompare(right.url) ||
-      left.rule.localeCompare(right.rule),
-  );
-
-  return issues;
-}

package/src/crawler.js

@@ -1,306 +0,0 @@
-import { createHash } from "node:crypto";
-import { extractSeoData } from "./html.js";
-import { createRequestGate } from "./request-gate.js";
-import { isAllowedByRobots, parseRobots } from "./robots.js";
-import { loadSitemapUrls } from "./sitemap.js";
-import {
-  isCrawlableUrl,
-  isSameOrigin,
-  normalizeUrl,
-} from "./urls.js";
-
-const DEFAULT_USER_AGENT = "seo-crawl-audit/0.1.2";
-const MAX_HTML_BYTES = 5 * 1024 * 1024;
-const MAX_ROBOTS_BYTES = 512 * 1024;
-
-async function readBody(response, maxBytes) {
-  if (!response.body) {
-    return "";
-  }
-
-  const reader = response.body.getReader();
-  const chunks = [];
-  let total = 0;
-
-  while (true) {
-    const { done, value } = await reader.read();
-    if (done) {
-      break;
-    }
-
-    total += value.byteLength;
-    if (total > maxBytes) {
-      await reader.cancel();
-      throw new Error(`response exceeds ${maxBytes} bytes`);
-    }
-    chunks.push(Buffer.from(value));
-  }
-
-  return Buffer.concat(chunks).toString("utf8");
-}
-
-async function loadRobots(startUrl, options) {
-  const robotsUrl = new URL("/robots.txt", startUrl).href;
-
-  try {
-    await options.requestGate();
-    const response = await fetch(robotsUrl, {
-      headers: {
-        accept: "text/plain,*/*;q=0.1",
-        "user-agent": options.userAgent,
-      },
-      redirect: "follow",
-      signal: AbortSignal.timeout(options.timeout),
-    });
-    const body = await readBody(response, MAX_ROBOTS_BYTES);
-
-    return {
-      url: robotsUrl,
-      status: response.status,
-      body,
-      sha256: createHash("sha256").update(body).digest("hex"),
-      rules: response.ok ? parseRobots(body, options.userAgent) : [],
-      denyAll: response.status === 401 || response.status === 403,
-      error: null,
-    };
-  } catch (error) {
-    return {
-      url: robotsUrl,
-      status: null,
-      body: "",
-      sha256: null,
-      rules: [],
-      denyAll: false,
-      error: error.message,
-    };
-  }
-}
-
-async function fetchPage(requestUrl, options) {
-  if (
-    options.respectRobots &&
-    !isAllowedByRobots(requestUrl, options.robots)
-  ) {
-    return {
-      url: requestUrl,
-      finalUrl: null,
-      status: null,
-      contentType: null,
-      blockedByRobots: true,
-      error: null,
-      title: null,
-      description: null,
-      canonical: null,
-      robots: null,
-      lang: null,
-      h1Count: 0,
-      openGraph: { title: null, description: null, image: null },
-      links: [],
-    };
-  }
-
-  try {
-    await options.requestGate();
-    const response = await fetch(requestUrl, {
-      headers: {
-        accept: "text/html,application/xhtml+xml;q=0.9,*/*;q=0.1",
-        "user-agent": options.userAgent,
-      },
-      redirect: "follow",
-      signal: AbortSignal.timeout(options.timeout),
-    });
-    const contentType = response.headers.get("content-type") ?? "";
-    const isHtml =
-      contentType.includes("text/html") ||
-      contentType.includes("application/xhtml+xml");
-    const finalUrl = normalizeUrl(response.url, undefined, {
-      includeQuery: options.includeQuery,
-    });
-
-    let seo = {
-      title: null,
-      description: null,
-      canonical: null,
-      robots: null,
-      lang: null,
-      h1Count: 0,
-      openGraph: { title: null, description: null, image: null },
-      links: [],
-    };
-
-    if (isHtml) {
-      const body = await readBody(response, MAX_HTML_BYTES);
-      seo = extractSeoData(body);
-    } else {
-      await response.body?.cancel();
-    }
-
-    const canonical = seo.canonical
-      ? normalizeUrl(seo.canonical, finalUrl ?? requestUrl, {
-          includeQuery: true,
-        })
-      : null;
-    const links = [
-      ...new Set(
-        seo.links
-          .map((link) =>
-            normalizeUrl(link, finalUrl ?? requestUrl, {
-              includeQuery: options.includeQuery,
-            }),
-          )
-          .filter(Boolean),
-      ),
-    ];
-
-    return {
-      url: requestUrl,
-      finalUrl,
-      status: response.status,
-      contentType: contentType || null,
-      blockedByRobots: false,
-      error: null,
-      ...seo,
-      canonical,
-      links,
-    };
-  } catch (error) {
-    return {
-      url: requestUrl,
-      finalUrl: null,
-      status: null,
-      contentType: null,
-      blockedByRobots: false,
-      error: error.message,
-      title: null,
-      description: null,
-      canonical: null,
-      robots: null,
-      lang: null,
-      h1Count: 0,
-      openGraph: { title: null, description: null, image: null },
-      links: [],
-    };
-  }
-}
-
-function withDefaults(options = {}) {
-  const requestDelay = options.requestDelay ?? 100;
-
-  return {
-    maxPages: options.maxPages ?? 100,
-    concurrency: options.concurrency ?? 5,
-    timeout: options.timeout ?? 10_000,
-    includeQuery: options.includeQuery ?? false,
-    respectRobots: options.respectRobots ?? true,
-    sitemap: options.sitemap ?? null,
-    sitemapData: options.sitemapData ?? null,
-    userAgent: options.userAgent ?? DEFAULT_USER_AGENT,
-    requestDelay,
-    requestGate:
-      options.requestGate ?? createRequestGate(requestDelay),
-  };
-}
-
-export async function fetchPages(urls, rawOptions = {}) {
-  const options = withDefaults(rawOptions);
-  const robots =
-    rawOptions.robots ?? (await loadRobots(urls[0], options));
-  const pages = [];
-  const cached = new Map(
-    (rawOptions.cachedPages ?? []).map((page) => [page.url, page]),
-  );
-
-  for (let index = 0; index < urls.length; index += options.concurrency) {
-    const batch = urls.slice(index, index + options.concurrency);
-    const freshUrls = batch.filter((url) => !cached.has(url));
-    const freshResults = await Promise.all(
-      freshUrls.map((url) => fetchPage(url, { ...options, robots })),
-    );
-    await rawOptions.onBatch?.(freshResults);
-    const fresh = new Map(freshResults.map((page) => [page.url, page]));
-    const results = batch.map((url) => cached.get(url) ?? fresh.get(url));
-    pages.push(...results);
-    rawOptions.onProgress?.(pages.length, urls.length);
-  }
-
-  return { pages, robots };
-}
-
-export async function crawlSite(inputUrl, rawOptions = {}) {
-  const options = withDefaults(rawOptions);
-  const startUrl = normalizeUrl(inputUrl, undefined, {
-    includeQuery: options.includeQuery,
-  });
-
-  if (!startUrl) {
-    throw new Error(`invalid start URL: ${inputUrl}`);
-  }
-
-  const origin = new URL(startUrl).origin;
-  const robots = await loadRobots(startUrl, options);
-  const sitemap = options.sitemapData ?? (options.sitemap
-    ? await loadSitemapUrls(options.sitemap, {
-        ...options,
-        maxUrls: options.sitemapMaxUrls ?? 50_000,
-        siteOrigin: origin,
-      })
-    : null);
-  const queue = [
-    ...new Set([startUrl, ...(sitemap?.urls ?? [])]),
-  ].slice(0, options.maxPages);
-  const scheduled = new Set(queue);
-  const cached = new Map(
-    (rawOptions.cachedPages ?? []).map((page) => [page.url, page]),
-  );
-  const pages = [];
-  let cursor = 0;
-  const maxQueueSize = Math.max(options.maxPages * 10, options.maxPages);
-
-  while (cursor < queue.length && pages.length < options.maxPages) {
-    const remaining = options.maxPages - pages.length;
-    const batch = queue.slice(
-      cursor,
-      cursor + Math.min(options.concurrency, remaining),
-    );
-    cursor += batch.length;
-
-    const freshUrls = batch.filter((url) => !cached.has(url));
-    const freshResults = await Promise.all(
-      freshUrls.map((url) => fetchPage(url, { ...options, robots })),
-    );
-    await rawOptions.onBatch?.(freshResults);
-    const fresh = new Map(freshResults.map((page) => [page.url, page]));
-    const results = batch.map((url) => cached.get(url) ?? fresh.get(url));
-    pages.push(...results);
-    rawOptions.onProgress?.(pages.length, options.maxPages);
-
-    for (const page of results) {
-      for (const link of page.links ?? []) {
-        if (
-          scheduled.size >= maxQueueSize ||
-          scheduled.has(link) ||
-          !isSameOrigin(link, origin) ||
-          !isCrawlableUrl(link)
-        ) {
-          continue;
-        }
-        scheduled.add(link);
-        queue.push(link);
-      }
-    }
-  }
-
-  pages.sort((left, right) => left.url.localeCompare(right.url));
-
-  return {
-    startUrl,
-    pages,
-    robots,
-    sitemap,
-    truncated:
-      cursor < queue.length ||
-      (sitemap ? sitemap.urls.length > pages.length : false) ||
-      (sitemap?.truncated ?? false),
-    options,
-  };
-}