similarbuild 0.1.0

package/templates/skills/sb-crawl-and-list/scripts/lib/blocklist-filter.mjs

@@ -0,0 +1,176 @@
+// blocklist-filter.mjs — Pure URL filtering. No I/O.
+//
+// Encodes the rule that a SimilarBuild page list must never include:
+//
+//   - Auth-gated pages (/account, /login, /admin, etc.) — irrelevant for
+//     visual cloning and many will 401/403.
+//   - Cart / checkout — dynamic, session-bound, and typically unique enough
+//     that a clone would mislead more than it informs.
+//   - API / framework internals (/api/**, /_next/**, /_nuxt/**) — not visual.
+//   - Tracking-only URLs (?utm_*, ?fbclid, ?gclid, ?mc_eid). These ARE
+//     duplicates of the canonical URL with a parameter; we drop the params
+//     and let dedup handle the rest.
+//   - RSS/Atom feeds — not pages, will not render.
+//
+// Custom blocklist (--blocklist) is additive: user prefixes are layered on
+// top of the defaults, never replace them. That way you can't accidentally
+// allow /admin by passing a too-narrow custom list.
+
+const DEFAULT_PATH_PREFIXES = [
+  '/cart',
+  '/checkout',
+  '/account',
+  '/login',
+  '/register',
+  '/signin',
+  '/signup',
+  '/sign-in',
+  '/sign-up',
+  '/logout',
+  '/admin',
+  '/wp-admin',
+  '/wp-login.php',
+  '/api/',
+  '/_next/',
+  '/_nuxt/',
+  '/feed',
+  '/rss',
+  '/atom',
+]
+
+// Tracking params we strip. Any URL whose ONLY querystring content is these
+// gets normalized to the bare path.
+const TRACKING_PARAMS = new Set([
+  'utm_source',
+  'utm_medium',
+  'utm_campaign',
+  'utm_term',
+  'utm_content',
+  'utm_id',
+  'utm_name',
+  'fbclid',
+  'gclid',
+  'gbraid',
+  'wbraid',
+  'mc_eid',
+  'mc_cid',
+  'msclkid',
+  'yclid',
+  '_ga',
+  'ref',
+  'ref_src',
+])
+
+function stripTracking(u) {
+  const params = u.searchParams
+  for (const k of [...params.keys()]) {
+    if (TRACKING_PARAMS.has(k.toLowerCase()) || k.toLowerCase().startsWith('utm_')) {
+      params.delete(k)
+    }
+  }
+  return u
+}
+
+// Normalize a URL for dedupe + filtering. Returns null if the URL is invalid,
+// off-origin, or has an unsupported scheme. `originHost` is the lower-cased
+// hostname of the root URL.
+export function normalizeUrl(url, originHost) {
+  try {
+    const u = new URL(url)
+    if (!/^https?:$/.test(u.protocol)) return null
+    if (originHost && u.hostname.toLowerCase() !== originHost) return null
+
+    // Drop fragment — anchors are intra-page navigation, not separate pages.
+    u.hash = ''
+    // Strip tracking params.
+    stripTracking(u)
+    // Sort remaining params for stable dedupe ordering.
+    const sorted = [...u.searchParams.entries()].sort(([a], [b]) => a.localeCompare(b))
+    u.search = ''
+    for (const [k, v] of sorted) u.searchParams.append(k, v)
+
+    // Normalize trailing slash on the pathname (except for root). This is
+    // dedupe-only: `/foo` and `/foo/` are the same page on 99% of sites.
+    if (u.pathname.length > 1 && u.pathname.endsWith('/')) {
+      u.pathname = u.pathname.slice(0, -1)
+    }
+    return u.toString()
+  } catch {
+    return null
+  }
+}
+
+// Check if a normalized URL matches any path prefix in the blocklist. The
+// match is on lowercased pathname only — querystrings can't trigger blocks.
+export function matchesBlocklist(url, blocklistPrefixes) {
+  let pathname
+  try {
+    pathname = new URL(url).pathname.toLowerCase()
+  } catch {
+    return false
+  }
+  // Special-case: exact path matches like '/login' (no slash) shouldn't
+  // accidentally match '/login-info'. Use boundary-aware matching: the
+  // prefix must end at a slash, end-of-path, or a non-word boundary.
+  for (const prefix of blocklistPrefixes) {
+    const p = prefix.toLowerCase()
+    if (pathname === p) return true
+    if (p.endsWith('/')) {
+      if (pathname.startsWith(p)) return true
+    } else {
+      if (pathname.startsWith(p + '/')) return true
+      // Files/exact endpoints like /wp-login.php
+      if (pathname === p) return true
+    }
+  }
+  return false
+}
+
+// .xml extensions — block every *.xml EXCEPT the one we used to crawl. The
+// crawler passes the sitemap URL it consumed (if any) so we don't accidentally
+// drop sitemap.xml itself if it appears in <loc> entries (which would be weird
+// but does happen on misconfigured sites).
+export function isXmlOrFeed(url, sitemapUrl) {
+  try {
+    const u = new URL(url)
+    const path = u.pathname.toLowerCase()
+    if (sitemapUrl) {
+      try {
+        const su = new URL(sitemapUrl)
+        if (su.pathname.toLowerCase() === path && su.hostname === u.hostname) {
+          return false
+        }
+      } catch {
+        /* fall through */
+      }
+    }
+    return path.endsWith('.xml') || path.endsWith('.rss') || path.endsWith('.atom')
+  } catch {
+    return false
+  }
+}
+
+export function buildBlocklist(customBlocklist) {
+  const custom = (customBlocklist || '')
+    .split(',')
+    .map((s) => s.trim())
+    .filter(Boolean)
+  return [...DEFAULT_PATH_PREFIXES, ...custom]
+}
+
+// Run the full pipeline for one URL. Returns:
+//   { keep: true,  url: normalized }                  — pass through
+//   { keep: false, reason: 'invalid'|'off-origin'|... } — drop
+export function filterUrl(url, { originHost, blocklistPrefixes, sitemapUrl }) {
+  const normalized = normalizeUrl(url, originHost)
+  if (!normalized) return { keep: false, reason: 'invalid-or-off-origin' }
+  if (matchesBlocklist(normalized, blocklistPrefixes)) {
+    return { keep: false, reason: 'blocklisted' }
+  }
+  if (isXmlOrFeed(normalized, sitemapUrl)) {
+    return { keep: false, reason: 'xml-or-feed' }
+  }
+  return { keep: true, url: normalized }
+}
+
+export const __defaults = { DEFAULT_PATH_PREFIXES, TRACKING_PARAMS }

package/templates/skills/sb-crawl-and-list/scripts/lib/fallback-crawler.mjs

@@ -0,0 +1,107 @@
+// fallback-crawler.mjs — Same-origin BFS using fetch + cheerio when sitemap
+// is absent or unparseable. Pure-ish: cheerio + fetcher are injected, no
+// process I/O. The crawler stops at maxDepth and maxPages and reports every
+// URL it discovered (including the ones over the cap, so the orchestrator
+// can warn).
+//
+// Why fetch + cheerio not chromium:
+//   - Discovery doesn't need rendering. Static <a href> tags reveal site
+//     structure on > 90% of WordPress / Shopify / Webflow / static sites.
+//   - Chromium adds 250MB install + ~5s startup per page; for 100 pages
+//     that's a 10-minute crawl when fetch finishes in 30s.
+//   - sb-inspect-live owns the chromium budget — we keep it for the actual
+//     visual capture.
+//
+// Trade-off accepted: pure-SPA sites (Next.js client-routed, fully
+// JS-rendered) return only the entry shell. We detect that ("0–1 internal
+// links found across the whole crawl") and surface a clear warning so the
+// user knows to supply a sitemap manually.
+
+import { filterUrl } from './blocklist-filter.mjs'
+
+// fetcher signature: async (url) => { ok, status, text? }
+// loadHtml signature: (html) => $ (cheerio instance)
+export async function fallbackCrawl({
+  rootUrl,
+  fetcher,
+  loadHtml,
+  maxDepth,
+  maxPages,
+  blocklistPrefixes,
+  sitemapUrl, // pass-through so isXmlOrFeed knows what to keep
+}) {
+  const origin = new URL(rootUrl)
+  const originHost = origin.hostname.toLowerCase()
+  const startUrl = origin.toString()
+
+  const visited = new Set()
+  const queue = [{ url: startUrl, depth: 0 }]
+  const results = [] // { url, depth }
+  const warnings = []
+
+  // Track the very first page's link count separately — that's our SPA
+  // detector. If the root yields 0–1 internal hrefs and total results <= 1
+  // at the end, the site is almost certainly client-rendered.
+  let rootInternalHrefCount = null
+
+  while (queue.length > 0 && results.length < maxPages) {
+    const { url, depth } = queue.shift()
+    const filtered = filterUrl(url, { originHost, blocklistPrefixes, sitemapUrl })
+    if (!filtered.keep) continue
+    const norm = filtered.url
+    if (visited.has(norm)) continue
+    visited.add(norm)
+
+    const fetched = await fetcher(norm)
+    if (!fetched.ok) {
+      warnings.push(`crawl-fetch-failed:${norm}:${fetched.status || 0}`)
+      continue
+    }
+
+    // The page itself counts even if it yields no further links.
+    results.push({ url: norm, depth })
+
+    if (depth >= maxDepth) continue
+
+    let $
+    try {
+      $ = loadHtml(fetched.text || '')
+    } catch (err) {
+      warnings.push(`crawl-parse-failed:${norm}:${err.message || 'unknown'}`)
+      continue
+    }
+
+    const hrefs = []
+    $('a[href]').each((_, el) => {
+      const raw = $(el).attr('href')
+      if (raw) hrefs.push(raw)
+    })
+
+    if (depth === 0) rootInternalHrefCount = 0
+
+    for (const raw of hrefs) {
+      let abs
+      try {
+        abs = new URL(raw, norm).toString()
+      } catch {
+        continue
+      }
+      const f = filterUrl(abs, { originHost, blocklistPrefixes, sitemapUrl })
+      if (!f.keep) continue
+      if (depth === 0) rootInternalHrefCount += 1
+      if (visited.has(f.url)) continue
+      // Don't enqueue if already queued (cheap check by scanning queue is
+      // O(n) but n is bounded by maxPages so it's fine).
+      if (queue.some((q) => q.url === f.url)) continue
+      queue.push({ url: f.url, depth: depth + 1 })
+    }
+  }
+
+  // SPA detector: tiny result set AND the entry page had ≤ 1 internal link.
+  // (We only signal the warning — the orchestrator decides what to do.)
+  if (results.length <= 1 && (rootInternalHrefCount ?? 0) <= 1) {
+    warnings.push('spa-suspected')
+  }
+
+  return { results, warnings, totalDiscovered: results.length + queue.length }
+}

package/templates/skills/sb-crawl-and-list/scripts/lib/page-classifier.mjs

@@ -0,0 +1,89 @@
+// page-classifier.mjs — URL → page-type heuristic via path inspection. Pure.
+//
+// Used by `/build-site` to bucket pages so the user sees a structured table
+// (e.g. "5 PDPs, 3 collections, 1 home") before confirming. The classification
+// drives nothing fatal — `other` is a valid type — but the better the
+// guesses, the less the orchestrator nags the user about reordering.
+//
+// Heuristics are intentionally Shopify-flavoured (most common SimilarBuild
+// target) but match common WordPress/Webflow paths too. Patterns are checked
+// top-down; first match wins. No regex slowness — these are literal substring
+// checks against the lowercased pathname.
+
+const TYPE_RULES = [
+  // Home — only the bare root. Trailing slash handled by normalizing pathname
+  // to "/" before matching.
+  { type: 'home', test: (p) => p === '/' || p === '' },
+
+  // Product pages — Shopify (`/products/`), WooCommerce / generic
+  // (`/product/`), some BigCommerce stores (`/product/`).
+  { type: 'pdp', test: (p) => p.startsWith('/products/') || p.startsWith('/product/') },
+
+  // Collections — Shopify (`/collections/`), some PrestaShop (`/category/`),
+  // WooCommerce (`/product-category/`).
+  {
+    type: 'collection',
+    test: (p) =>
+      p.startsWith('/collections/') ||
+      p.startsWith('/collection/') ||
+      p.startsWith('/category/') ||
+      p.startsWith('/product-category/'),
+  },
+
+  // Contact — must match before generic /pages/* fallthrough.
+  {
+    type: 'contact',
+    test: (p) => /^\/pages\/contact(\b|\/|-)/.test(p) || p === '/contact' || p.startsWith('/contact/'),
+  },
+
+  // About — same shape as contact.
+  {
+    type: 'about',
+    test: (p) => /^\/pages\/about(\b|\/|-)/.test(p) || p === '/about' || p.startsWith('/about/'),
+  },
+
+  // Policy pages — Shopify uses /policies/, generic stores use named pages.
+  {
+    type: 'policy',
+    test: (p) =>
+      p.startsWith('/policies/') ||
+      /^\/pages\/(privacy|terms|refund|shipping|return)(\b|\/|-)/.test(p) ||
+      /^\/(privacy|terms|refund|shipping|return)(\b|\/|-)/.test(p),
+  },
+
+  // Blog index + posts. Shopify: /blogs/, WordPress: /blog/, generic /posts/.
+  {
+    type: 'blog',
+    test: (p) =>
+      p.startsWith('/blogs/') ||
+      p.startsWith('/blog/') ||
+      p === '/blog' ||
+      p.startsWith('/posts/') ||
+      p === '/news' ||
+      p.startsWith('/news/'),
+  },
+]
+
+// Normalize a URL string to its lowercase pathname. Falls back to the input
+// if it's not parseable (caller may pass a path-only string for testing).
+function pathnameOf(url) {
+  try {
+    const u = new URL(url)
+    let p = u.pathname.toLowerCase()
+    // Treat trailing slash as equal to non-trailing (except for "/")
+    if (p.length > 1 && p.endsWith('/')) p = p.slice(0, -1)
+    return p
+  } catch {
+    let p = String(url).toLowerCase()
+    if (p.length > 1 && p.endsWith('/')) p = p.slice(0, -1)
+    return p
+  }
+}
+
+export function classifyUrl(url) {
+  const path = pathnameOf(url)
+  for (const rule of TYPE_RULES) {
+    if (rule.test(path)) return rule.type
+  }
+  return 'other'
+}

package/templates/skills/sb-crawl-and-list/scripts/lib/sitemap-parser.mjs

@@ -0,0 +1,118 @@
+// sitemap-parser.mjs — Pure XML → URL extraction. No I/O. Tested in isolation.
+//
+// Sitemap protocol (sitemaps.org) is small enough that a regex-driven parser
+// beats pulling in xml2js for an install-time cost we don't need. We accept:
+//
+//   - <urlset> with <url><loc>…</loc></url> (the common case)
+//   - <sitemapindex> with <sitemap><loc>…</loc></sitemap> (recurse one level)
+//
+// We do NOT validate XML well-formedness — sitemaps in the wild are full of
+// stray BOMs, missing closing tags, and CDATA quirks. Strict validation would
+// reject sitemaps Google's crawler happily eats. Instead: extract every <loc>
+// payload, decode entities, trim, dedupe, and let downstream filtering decide.
+//
+// Returns: { urlSetUrls, sitemapIndexUrls, malformed }
+
+const ENTITY_MAP = {
+  '&amp;': '&',
+  '&lt;': '<',
+  '&gt;': '>',
+  '&quot;': '"',
+  '&apos;': "'",
+}
+
+function decodeEntities(s) {
+  return s
+    .replace(/&(amp|lt|gt|quot|apos);/g, (m) => ENTITY_MAP[m] || m)
+    .replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(parseInt(n, 10)))
+    .replace(/&#x([0-9a-f]+);/gi, (_, n) => String.fromCodePoint(parseInt(n, 16)))
+}
+
+function extractLocs(xml) {
+  const out = []
+  // Capture <loc>…</loc> non-greedy. Allow whitespace + CDATA wrapping.
+  const re = /<loc\b[^>]*>\s*(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?\s*<\/loc\s*>/gi
+  let m
+  while ((m = re.exec(xml)) !== null) {
+    const raw = m[1].trim()
+    if (!raw) continue
+    out.push(decodeEntities(raw))
+  }
+  return out
+}
+
+// Distinguish between <urlset> (page URLs) and <sitemapindex> (more sitemaps).
+// Sitemap protocol allows nested sitemapindex but we recurse only one level —
+// real-world sites that nest deeper are rare and orchestrator can pass a
+// specific child sitemap via --sitemap-path.
+function detectKind(xml) {
+  if (/<\s*sitemapindex\b/i.test(xml)) return 'index'
+  if (/<\s*urlset\b/i.test(xml)) return 'urlset'
+  // Some hand-rolled sitemaps skip the wrapper entirely. If we find <loc>
+  // tags at all, treat as a flat urlset.
+  if (/<loc\b/i.test(xml)) return 'urlset'
+  return 'unknown'
+}
+
+export function parseSitemap(xml) {
+  if (typeof xml !== 'string' || xml.length === 0) {
+    return { kind: 'unknown', urlSetUrls: [], sitemapIndexUrls: [], malformed: true }
+  }
+  // Strip BOM if present — common from CMS-exported sitemaps.
+  if (xml.charCodeAt(0) === 0xfeff) xml = xml.slice(1)
+
+  const kind = detectKind(xml)
+  if (kind === 'unknown') {
+    return { kind, urlSetUrls: [], sitemapIndexUrls: [], malformed: true }
+  }
+  const locs = extractLocs(xml)
+  if (kind === 'index') {
+    return { kind, urlSetUrls: [], sitemapIndexUrls: locs, malformed: false }
+  }
+  return { kind, urlSetUrls: locs, sitemapIndexUrls: [], malformed: false }
+}
+
+// Async fetch + parse with one level of sitemapindex recursion. Fetcher is
+// injected so this stays pure-ish and testable. Returns merged URL list +
+// any per-child errors as warnings.
+//
+// fetcher signature: async (url) => { ok, status, text? }
+export async function fetchAndParseSitemap(url, fetcher, { maxChildren = 50 } = {}) {
+  const warnings = []
+  const collected = []
+
+  const top = await fetcher(url)
+  if (!top.ok) {
+    return { urls: [], warnings: [`sitemap-fetch-failed:${top.status || 0}`] }
+  }
+  const parsed = parseSitemap(top.text || '')
+  if (parsed.malformed) {
+    return { urls: [], warnings: ['sitemap-malformed'] }
+  }
+  if (parsed.kind === 'urlset') {
+    return { urls: parsed.urlSetUrls, warnings }
+  }
+
+  // sitemapindex — fetch each child sitemap up to maxChildren.
+  const children = parsed.sitemapIndexUrls.slice(0, maxChildren)
+  if (parsed.sitemapIndexUrls.length > maxChildren) {
+    warnings.push(`sitemap-index-truncated:${parsed.sitemapIndexUrls.length}-of-${maxChildren}`)
+  }
+  for (const childUrl of children) {
+    const r = await fetcher(childUrl)
+    if (!r.ok) {
+      warnings.push(`sitemap-child-failed:${childUrl}:${r.status || 0}`)
+      continue
+    }
+    const childParsed = parseSitemap(r.text || '')
+    if (childParsed.malformed) {
+      warnings.push(`sitemap-child-malformed:${childUrl}`)
+      continue
+    }
+    // Treat any nested sitemapindex as flat — don't recurse a second level.
+    const childUrls =
+      childParsed.kind === 'urlset' ? childParsed.urlSetUrls : []
+    collected.push(...childUrls)
+  }
+  return { urls: collected, warnings }
+}

package/templates/skills/sb-crawl-and-list/scripts/tests/test-blocklist-filter.mjs

@@ -0,0 +1,204 @@
+#!/usr/bin/env node
+// Tests for lib/blocklist-filter.mjs — pure unit tests.
+
+import { strict as assert } from 'node:assert'
+import {
+  buildBlocklist,
+  filterUrl,
+  matchesBlocklist,
+  normalizeUrl,
+  isXmlOrFeed,
+  __defaults,
+} from '../lib/blocklist-filter.mjs'
+
+let passed = 0
+let failed = 0
+
+function test(name, fn) {
+  try {
+    fn()
+    process.stdout.write(`ok - ${name}\n`)
+    passed++
+  } catch (err) {
+    process.stdout.write(`not ok - ${name}\n  ${err.message}\n`)
+    failed++
+  }
+}
+
+// ─── normalizeUrl ───────────────────────────────────────────────────────────
+
+test('normalizeUrl strips fragment', () => {
+  assert.equal(
+    normalizeUrl('https://example.com/page#section', 'example.com'),
+    'https://example.com/page',
+  )
+})
+
+test('normalizeUrl strips utm_* tracking params', () => {
+  assert.equal(
+    normalizeUrl('https://example.com/page?utm_source=twitter&utm_medium=social', 'example.com'),
+    'https://example.com/page',
+  )
+})
+
+test('normalizeUrl strips fbclid/gclid/mc_eid', () => {
+  assert.equal(
+    normalizeUrl('https://example.com/p?fbclid=123', 'example.com'),
+    'https://example.com/p',
+  )
+  assert.equal(
+    normalizeUrl('https://example.com/p?gclid=abc', 'example.com'),
+    'https://example.com/p',
+  )
+  assert.equal(
+    normalizeUrl('https://example.com/p?mc_eid=z', 'example.com'),
+    'https://example.com/p',
+  )
+})
+
+test('normalizeUrl preserves non-tracking params', () => {
+  assert.equal(
+    normalizeUrl('https://example.com/products?variant=red', 'example.com'),
+    'https://example.com/products?variant=red',
+  )
+})
+
+test('normalizeUrl rejects off-origin URLs', () => {
+  assert.equal(normalizeUrl('https://other.com/foo', 'example.com'), null)
+})
+
+test('normalizeUrl rejects non-http(s) schemes', () => {
+  assert.equal(normalizeUrl('mailto:hello@example.com', 'example.com'), null)
+  assert.equal(normalizeUrl('ftp://example.com/file', 'example.com'), null)
+})
+
+test('normalizeUrl strips trailing slash on non-root pathnames', () => {
+  assert.equal(
+    normalizeUrl('https://example.com/products/foo/', 'example.com'),
+    'https://example.com/products/foo',
+  )
+})
+
+test('normalizeUrl preserves root slash', () => {
+  assert.equal(normalizeUrl('https://example.com/', 'example.com'), 'https://example.com/')
+})
+
+test('normalizeUrl returns null on garbage input', () => {
+  assert.equal(normalizeUrl('not-a-url', 'example.com'), null)
+})
+
+// ─── matchesBlocklist ───────────────────────────────────────────────────────
+
+test('matchesBlocklist matches /admin and /admin/foo', () => {
+  const list = ['/admin']
+  assert.ok(matchesBlocklist('https://example.com/admin', list))
+  assert.ok(matchesBlocklist('https://example.com/admin/users', list))
+})
+
+test('matchesBlocklist does NOT falsely match /admin-tools as /admin', () => {
+  assert.ok(!matchesBlocklist('https://example.com/admin-tools', ['/admin']))
+})
+
+test('matchesBlocklist matches /api/ prefix', () => {
+  assert.ok(matchesBlocklist('https://example.com/api/v1/users', ['/api/']))
+})
+
+test('matchesBlocklist matches exact /wp-login.php file', () => {
+  assert.ok(matchesBlocklist('https://example.com/wp-login.php', ['/wp-login.php']))
+})
+
+test('matchesBlocklist is case-insensitive on path', () => {
+  assert.ok(matchesBlocklist('https://example.com/Admin', ['/admin']))
+})
+
+// ─── isXmlOrFeed ────────────────────────────────────────────────────────────
+
+test('isXmlOrFeed flags arbitrary .xml URLs', () => {
+  assert.ok(isXmlOrFeed('https://example.com/foo.xml'))
+})
+
+test('isXmlOrFeed allows the active sitemap to pass through', () => {
+  assert.ok(
+    !isXmlOrFeed('https://example.com/sitemap.xml', 'https://example.com/sitemap.xml'),
+  )
+})
+
+test('isXmlOrFeed flags /feed and rss/atom paths via prefix logic separately (here just .xml)', () => {
+  assert.ok(isXmlOrFeed('https://example.com/feed.atom'))
+  assert.ok(isXmlOrFeed('https://example.com/posts.rss'))
+})
+
+// ─── buildBlocklist ─────────────────────────────────────────────────────────
+
+test('buildBlocklist includes all defaults', () => {
+  const list = buildBlocklist()
+  for (const p of __defaults.DEFAULT_PATH_PREFIXES) {
+    assert.ok(list.includes(p), `missing default ${p}`)
+  }
+})
+
+test('buildBlocklist appends custom prefixes additively', () => {
+  const list = buildBlocklist('/preview, /internal,/staging')
+  assert.ok(list.includes('/preview'))
+  assert.ok(list.includes('/internal'))
+  assert.ok(list.includes('/staging'))
+  // Defaults still present
+  assert.ok(list.includes('/admin'))
+})
+
+test('buildBlocklist tolerates empty/whitespace custom string', () => {
+  const list = buildBlocklist('   ,, ')
+  assert.deepEqual(list, [...__defaults.DEFAULT_PATH_PREFIXES])
+})
+
+// ─── filterUrl (full pipeline) ──────────────────────────────────────────────
+
+const ctx = {
+  originHost: 'example.com',
+  blocklistPrefixes: buildBlocklist('/preview'),
+  sitemapUrl: 'https://example.com/sitemap.xml',
+}
+
+test('filterUrl keeps a clean product URL', () => {
+  const r = filterUrl('https://example.com/products/foo', ctx)
+  assert.equal(r.keep, true)
+  assert.equal(r.url, 'https://example.com/products/foo')
+})
+
+test('filterUrl normalizes tracking-laden URL before checking', () => {
+  const r = filterUrl('https://example.com/products/foo?utm_source=x#hash', ctx)
+  assert.equal(r.keep, true)
+  assert.equal(r.url, 'https://example.com/products/foo')
+})
+
+test('filterUrl drops /admin via default blocklist', () => {
+  const r = filterUrl('https://example.com/admin/users', ctx)
+  assert.equal(r.keep, false)
+  assert.equal(r.reason, 'blocklisted')
+})
+
+test('filterUrl drops /preview via custom blocklist', () => {
+  const r = filterUrl('https://example.com/preview/page', ctx)
+  assert.equal(r.keep, false)
+  assert.equal(r.reason, 'blocklisted')
+})
+
+test('filterUrl drops off-origin URL', () => {
+  const r = filterUrl('https://other.com/foo', ctx)
+  assert.equal(r.keep, false)
+  assert.equal(r.reason, 'invalid-or-off-origin')
+})
+
+test('filterUrl drops generic *.xml but keeps the active sitemap', () => {
+  const r = filterUrl('https://example.com/products.xml', ctx)
+  assert.equal(r.keep, false)
+  assert.equal(r.reason, 'xml-or-feed')
+  // Active sitemap URL is filtered as "keep" because we treat it as a non-page,
+  // i.e. the active sitemap shouldn't appear in URL list anyway. But isXmlOrFeed
+  // returns false for the active sitemap so the keep path runs:
+  const r2 = filterUrl('https://example.com/sitemap.xml', ctx)
+  assert.equal(r2.keep, true)
+})
+
+process.stdout.write(`\n${passed} passed, ${failed} failed\n`)
+process.exit(failed === 0 ? 0 : 1)