website-to-markdown 1.1.1 → 2.0.0

package/dist/sitemap/index.js

@@ -1,8 +1,19 @@
-import { escapeRegExp } from 'hanzio';
+import { getUniqueValues, isOneOf, trimTrailingSlash } from 'hanzio';
+import { isAllowedByRobots, parseRobotsTxt } from 'hanzio/sitemap';
 import { throughCrawlSnapshot } from '../crawl-snapshot.js';
+import { guardedFetch } from '../net-guard.js';
+import { decodeText, readBody } from '../read-body.js';
+import { BROWSER_USER_AGENT } from '../user-agent.js';
 const FETCH_TIMEOUT_MS = 10_000;
 const MAX_SITEMAP_FILES = 20;
 const MAX_URLS = 20_000;
+/** Google reads the first 500 KiB of a robots.txt and ignores the rest. */
+const MAX_ROBOTS_BYTES = 500 * 1024;
+/**
+ * Read cap per sitemap file. 20,000 URLs fit in a few MB; a longer file is
+ * read up to here and its complete entries used.
+ */
+const MAX_SITEMAP_BYTES = 10 * 1024 * 1024;
 /** Conventional sitemap locations, tried in order when robots.txt names none. */
 const SITEMAP_ROOT_PATHS = [
     '/sitemap.xml',
@@ -55,27 +66,35 @@ export function parseSitemapXml(xml) {
     }
     return { pages, nested };
 }
-function fetchText(url, accept) {
-    return throughCrawlSnapshot(`text ${accept} ${url}`, () => fetchTextLive(url, accept));
+function fetchText(url, request) {
+    return throughCrawlSnapshot(`text ${request.accept} ${url}`, () => fetchTextLive(url, request));
 }
-async function fetchTextLive(url, accept) {
+async function fetchTextLive(url, request) {
     try {
-        const response = await fetch(url, {
+        const response = await request.fetch(url, {
             signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
-            headers: { accept }
+            headers: { accept: request.accept, 'user-agent': BROWSER_USER_AGENT }
         });
-        if (!response.ok)
-            return null;
         const type = response.headers.get('content-type') ?? '';
         // A CMS that renders its 404 page for a missing file answers 200 with HTML.
-        if (type.includes('text/html'))
+        if (!response.ok || type.includes('text/html')) {
+            void response.body?.cancel().catch(() => { });
             return null;
-        return await response.text();
+        }
+        return decodeText((await readBody(response, request.maxBytes)).bytes);
     }
     catch {
         return null;
     }
 }
+const ROBOTS_REQUEST = {
+    accept: 'text/plain,*/*;q=0.8',
+    maxBytes: MAX_ROBOTS_BYTES
+};
+const SITEMAP_REQUEST = {
+    accept: 'application/xml,text/xml;q=0.9,*/*;q=0.8',
+    maxBytes: MAX_SITEMAP_BYTES
+};
 function originOf(websiteUrl) {
     try {
         return new URL(websiteUrl).origin;
@@ -90,13 +109,16 @@ function originOf(websiteUrl) {
  * here), and its rules. An HTML body or any non-200 means "no robots.txt";
  * never throws.
  */
-export async function fetchRobots(websiteUrl) {
+export async function fetchRobots(websiteUrl, options = {}) {
     const sitemaps = [];
     const origin = originOf(websiteUrl);
     if (!origin)
         return { sitemaps, rules: [] };
     const robotsUrl = `${origin}/robots.txt`;
-    const body = await fetchText(robotsUrl, 'text/plain,*/*;q=0.8');
+    const body = await fetchText(robotsUrl, {
+        ...ROBOTS_REQUEST,
+        fetch: options.fetch ?? guardedFetch
+    });
     if (!body)
         return { sitemaps, rules: [] };
     for (const match of body.matchAll(ROBOTS_SITEMAP_RE)) {
@@ -106,17 +128,18 @@ export async function fetchRobots(websiteUrl) {
         try {
             const absolute = new URL(value, robotsUrl);
             // Only what we could actually GET - `ftp:` and friends are noise.
-            if (absolute.protocol !== 'http:' && absolute.protocol !== 'https:')
+            if (!isOneOf(['http:', 'https:'], absolute.protocol))
                 continue;
-            const href = absolute.toString();
-            if (!sitemaps.includes(href))
-                sitemaps.push(href);
+            sitemaps.push(absolute.toString());
         }
         catch {
             // A malformed Sitemap: line must not cost us the other lines.
         }
     }
-    return { sitemaps, rules: parseRobotsRules(body) };
+    return {
+        sitemaps: getUniqueValues(sitemaps),
+        rules: parseRobotsRules(body)
+    };
 }
 /**
  * The `Allow`/`Disallow` lines of the `User-agent: *` groups. We read pages
@@ -125,81 +148,29 @@ export async function fetchRobots(websiteUrl) {
  * allows everything and adds no rule.
  */
 export function parseRobotsRules(body) {
-    const rules = [];
-    let agents = [];
-    let inRules = false;
-    for (const raw of body.split(/\r?\n/)) {
-        const match = /^\s*([a-z-]+)\s*:\s*(.*?)\s*$/i.exec(raw.replace(/#.*$/, ''));
-        if (!match)
-            continue;
-        const field = (match[1] ?? '').toLowerCase();
-        const value = match[2] ?? '';
-        if (field === 'user-agent') {
-            // A user-agent line after rules opens a new group.
-            if (inRules)
-                agents = [];
-            inRules = false;
-            agents.push(value.toLowerCase());
-            continue;
-        }
-        if (field !== 'allow' && field !== 'disallow')
-            continue;
-        inRules = true;
-        if (agents.includes('*') && value !== '')
-            rules.push({ allow: field === 'allow', pattern: value });
-    }
-    return rules;
-}
-/** Compiled once per pattern: the crawl asks for every link of every page. */
-const robotsPatternCache = new Map();
-function robotsPatternRegex(pattern) {
-    const cached = robotsPatternCache.get(pattern);
-    if (cached)
-        return cached;
-    const anchored = pattern.endsWith('$');
-    const body = (anchored ? pattern.slice(0, -1) : pattern)
-        .split('*')
-        .map((part) => escapeRegExp(part))
-        .join('.*');
-    const regex = new RegExp(`^${body}${anchored ? '$' : ''}`);
-    // Every site's patterns land here; a long-lived worker starts over rather than grow.
-    if (robotsPatternCache.size >= 5_000)
-        robotsPatternCache.clear();
-    robotsPatternCache.set(pattern, regex);
-    return regex;
+    const { allow, disallow } = parseRobotsTxt(body);
+    return [
+        ...allow.map((pattern) => ({ allow: true, pattern })),
+        ...disallow.map((pattern) => ({ allow: false, pattern }))
+    ];
 }
 /**
  * Whether robots.txt lets us read this URL: the longest matching rule wins and
  * `Allow` wins a tie, as Google reads it. Shopify disallows `/cart/`, and the
- * crawler had fetched `/cart/add?id=…&quantity=1` - a GET that fills a cart
- * (Good Habit audit §3.5).
+ * crawler had fetched `/cart/add?id=…&quantity=1` - a GET that fills a cart.
  */
 export function robotsAllows(rules, url) {
-    let path;
-    try {
-        const parsed = new URL(url);
-        path = parsed.pathname + parsed.search;
-    }
-    catch {
-        return true;
-    }
-    let best = null;
-    for (const rule of rules) {
-        if (!robotsPatternRegex(rule.pattern).test(path))
-            continue;
-        if (!best || rule.pattern.length > best.pattern.length)
-            best = rule;
-        else if (rule.pattern.length === best.pattern.length && rule.allow)
-            best = rule;
-    }
-    return best?.allow ?? true;
+    return isAllowedByRobots({
+        allow: rules.filter((rule) => rule.allow).map((rule) => rule.pattern),
+        disallow: rules.filter((rule) => !rule.allow).map((rule) => rule.pattern)
+    }, url);
 }
 /**
  * Reads a set of sitemap roots (following index nesting) into `found`.
  * Returns true when at least one of them yielded page URLs, so the caller can
  * stop trying further roots.
  */
-async function readSitemapTree(roots, found, seen) {
+async function readSitemapTree(roots, found, seen, fetch) {
     let readable = false;
     let level = [...roots];
     // One level of the index at a time, its files read side by side: a shop
@@ -214,7 +185,7 @@ async function readSitemapTree(roots, found, seen) {
             seen.add(sitemapUrl);
             batch.push(sitemapUrl);
         }
-        const bodies = await Promise.all(batch.map((sitemapUrl) => fetchText(sitemapUrl, 'application/xml,text/xml;q=0.9,*/*;q=0.8')));
+        const bodies = await Promise.all(batch.map((sitemapUrl) => fetchText(sitemapUrl, { ...SITEMAP_REQUEST, fetch })));
         const next = [];
         for (const xml of bodies) {
             if (!xml)
@@ -247,10 +218,11 @@ async function readSitemapTree(roots, found, seen) {
  * an error: link-following discovery is the source, sitemaps a supplement.
  * Never throws.
  */
-export async function discoverSitemapUrls(websiteUrl, robotsSitemaps) {
+export async function discoverSitemapUrls(websiteUrl, robotsSitemaps, options = {}) {
     const origin = originOf(websiteUrl);
     if (!origin)
         return [];
+    const fetch = options.fetch ?? guardedFetch;
     const found = new Map();
     // Shared across roots so a broken sitemap index cannot spend the file budget
     // several times over.
@@ -258,11 +230,11 @@ export async function discoverSitemapUrls(websiteUrl, robotsSitemaps) {
     // robots.txt may declare several sitemaps (WordPress splits per post type),
     // so all of them are read together - only the conventional roots compete.
     if (robotsSitemaps.length > 0 &&
-        (await readSitemapTree(robotsSitemaps, found, seen))) {
+        (await readSitemapTree(robotsSitemaps, found, seen, fetch))) {
         return [...found.values()];
     }
     for (const path of SITEMAP_ROOT_PATHS) {
-        if (await readSitemapTree([`${origin}${path}`], found, seen))
+        if (await readSitemapTree([`${origin}${path}`], found, seen, fetch))
             break;
     }
     return [...found.values()];
@@ -272,10 +244,10 @@ export function normalizeSitemapUrl(url) {
     try {
         const parsed = new URL(url);
         parsed.hash = '';
-        return parsed.toString().replace(/\/+$/, '');
+        return trimTrailingSlash(parsed.toString());
     }
     catch {
-        return url.replace(/\/+$/, '');
+        return trimTrailingSlash(url);
     }
 }
 //# sourceMappingURL=index.js.map

package/dist/sitemap/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/sitemap/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAA;AACrC,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAA;AAE3D,MAAM,gBAAgB,GAAG,MAAM,CAAA;AAC/B,MAAM,iBAAiB,GAAG,EAAE,CAAA;AAC5B,MAAM,QAAQ,GAAG,MAAM,CAAA;AAEvB,iFAAiF;AACjF,MAAM,kBAAkB,GAAG;IAC1B,cAAc;IACd,oBAAoB;IACpB,iBAAiB;CACjB,CAAA;AAED,MAAM,MAAM,GAAG,8BAA8B,CAAA;AAC7C,MAAM,YAAY,GAAG,0BAA0B,CAAA;AAC/C,MAAM,gBAAgB,GAAG,kCAAkC,CAAA;AAC3D,MAAM,YAAY,GAAG,6BAA6B,CAAA;AAClD,MAAM,UAAU,GAAG,qCAAqC,CAAA;AACxD,MAAM,iBAAiB,GAAG,iDAAiD,CAAA;AAE3E,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC;IAC5B,CAAC,OAAO,EAAE,GAAG,CAAC;IACd,CAAC,MAAM,EAAE,GAAG,CAAC;IACb,CAAC,MAAM,EAAE,GAAG,CAAC;IACb,CAAC,QAAQ,EAAE,GAAG,CAAC;IACf,CAAC,QAAQ,EAAE,GAAG,CAAC;IACf,CAAC,OAAO,EAAE,GAAG,CAAC;CACd,CAAC,CAAA;AAEF,gEAAgE;AAChE,SAAS,iBAAiB,CAAC,KAAa;IACvC,OAAO,KAAK,CAAC,OAAO,CACnB,gCAAgC,EAChC,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,MAAM,CAC9C,CAAA;AACF,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,eAAe,CAAC,GAAW;IAC1C,MAAM,KAAK,GAA8C,EAAE,CAAA;IAC3D,MAAM,MAAM,GAAa,EAAE,CAAA;IAE3B,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;QAChD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;QAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAC1C,IAAI,CAAC,GAAG;YAAE,SAAQ;QAClB,KAAK,CAAC,IAAI,CAAC;YACV,GAAG,EAAE,iBAAiB,CAAC,GAAG,CAAC;YAC3B,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,IAAI;SACrD,CAAC,CAAA;IACH,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACpD,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAC9C,IAAI,GAAG;YAAE,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAA;IAC7C,CAAC;IACD,6EAA6E;IAC7E,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/C,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1C,IAAI,KAAK,CAAC,CAAC,CAAC;gBACX,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;QACjE,CAAC;IACF,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAA;AACzB,CAAC;AAED,SAAS,SAAS,CAAC,GAAW,EAAE,MAAc;IAC7C,OAAO,oBAAoB,CAAC,QAAQ,MAAM,IAAI,GAAG,EAAE,EAAE,GAAG,EAAE,CACzD,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,CAC1B,CAAA;AACF,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,GAAW,EAAE,MAAc;IACvD,IAAI,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YACjC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,gBAAgB,CAAC;YAC7C,OAAO,EAAE,EAAE,MAAM,EAAE;SACnB,CAAC,CAAA;QACF,IAAI,CAAC,QAAQ,CAAC,EAAE;YAAE,OAAO,IAAI,CAAA;QAC7B,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAA;QACvD,4EAA4E;QAC5E,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC;YAAE,OAAO,IAAI,CAAA;QAC3C,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;IAC7B,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC;AAED,SAAS,QAAQ,CAAC,UAAkB;IACnC,IAAI,CAAC;QACJ,OAAO,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,MAAM,CAAA;IAClC,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,UAAkB;IACnD,MAAM,QAAQ,GAAa,EAAE,CAAA;IAC7B,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAA;IACnC,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,EAAE,CAAA;IAC3C,MAAM,SAAS,GAAG,GAAG,MAAM,aAAa,CAAA;IACxC,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,SAAS,EAAE,sBAAsB,CAAC,CAAA;IAC/D,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,EAAE,CAAA;IAEzC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;QACtD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACtB,IAAI,CAAC,KAAK;YAAE,SAAQ;QACpB,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;YAC1C,kEAAkE;YAClE,IAAI,QAAQ,CAAC,QAAQ,KAAK,OAAO,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ;gBAClE,SAAQ;YACT,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAA;YAChC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAClD,CAAC;QAAC,MAAM,CAAC;YACR,8DAA8D;QAC/D,CAAC;IACF,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAA;AACnD,CAAC;AAID;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC5C,MAAM,KAAK,GAAiB,EAAE,CAAA;IAC9B,IAAI,MAAM,GAAa,EAAE,CAAA;IACzB,IAAI,OAAO,GAAG,KAAK,CAAA;IAEnB,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,gCAAgC,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAA;QAC5E,IAAI,CAAC,KAAK;YAAE,SAAQ;QACpB,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAA;QAC5C,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;QAE5B,IAAI,KAAK,KAAK,YAAY,EAAE,CAAC;YAC5B,mDAAmD;YACnD,IAAI,OAAO;gBAAE,MAAM,GAAG,EAAE,CAAA;YACxB,OAAO,GAAG,KAAK,CAAA;YACf,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAA;YAChC,SAAQ;QACT,CAAC;QACD,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,UAAU;YAAE,SAAQ;QAEvD,OAAO,GAAG,IAAI,CAAA;QACd,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,KAAK,EAAE;YACvC,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,KAAK,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAA;IAC1D,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED,8EAA8E;AAC9E,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAkB,CAAA;AAEpD,SAAS,kBAAkB,CAAC,OAAe;IAC1C,MAAM,MAAM,GAAG,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAC9C,IAAI,MAAM;QAAE,OAAO,MAAM,CAAA;IAEzB,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;IACtC,MAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;SACtD,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;SACjC,IAAI,CAAC,IAAI,CAAC,CAAA;IACZ,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,IAAI,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IAC1D,qFAAqF;IACrF,IAAI,kBAAkB,CAAC,IAAI,IAAI,KAAK;QAAE,kBAAkB,CAAC,KAAK,EAAE,CAAA;IAChE,kBAAkB,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;IACtC,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,KAAmB,EAAE,GAAW;IAC5D,IAAI,IAAY,CAAA;IAChB,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;QAC3B,IAAI,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAA;IACvC,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,IAAI,IAAI,GAAsB,IAAI,CAAA;IAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAQ;QAC1D,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,IAAI,GAAG,IAAI,CAAA;aAC9D,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK;YACjE,IAAI,GAAG,IAAI,CAAA;IACb,CAAC;IACD,OAAO,IAAI,EAAE,KAAK,IAAI,IAAI,CAAA;AAC3B,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,eAAe,CAC7B,KAAe,EACf,KAA2D,EAC3D,IAAiB;IAEjB,IAAI,QAAQ,GAAG,KAAK,CAAA;IACpB,IAAI,KAAK,GAAG,CAAC,GAAG,KAAK,CAAC,CAAA;IAEtB,wEAAwE;IACxE,4EAA4E;IAC5E,2EAA2E;IAC3E,oEAAoE;IACpE,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,GAAG,QAAQ,EAAE,CAAC;QAClD,MAAM,KAAK,GAAa,EAAE,CAAA;QAC1B,KAAK,MAAM,UAAU,IAAI,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,IAAI,IAAI,iBAAiB;gBAAE,SAAQ;YACpE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;YACpB,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;QACvB,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CACxB,SAAS,CAAC,UAAU,EAAE,0CAA0C,CAAC,CACjE,CACD,CAAA;QACD,MAAM,IAAI,GAAa,EAAE,CAAA;QACzB,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YAC1B,IAAI,CAAC,GAAG;gBAAE,SAAQ;YAClB,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,eAAe,CAAC,GAAG,CAAC,CAAA;YAC9C,yEAAyE;YACzE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAQ;YACvD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;gBAAE,QAAQ,GAAG,IAAI,CAAA;YACrC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAA;YACpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBAC1B,IAAI,KAAK,CAAC,IAAI,IAAI,QAAQ;oBAAE,MAAK;gBACjC,uEAAuE;gBACvE,+CAA+C;gBAC/C,MAAM,GAAG,GAAG,mBAAmB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACzC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC;oBAAE,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;YAC1C,CAAC;QACF,CAAC;QACD,KAAK,GAAG,IAAI,CAAA;IACb,CAAC;IACD,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACxC,UAAkB,EAClB,cAAwB;IAExB,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAA;IACnC,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,CAAA;IACtB,MAAM,KAAK,GAAG,IAAI,GAAG,EAAmD,CAAA;IACxE,6EAA6E;IAC7E,sBAAsB;IACtB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAE9B,4EAA4E;IAC5E,0EAA0E;IAC1E,IACC,cAAc,CAAC,MAAM,GAAG,CAAC;QACzB,CAAC,MAAM,eAAe,CAAC,cAAc,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,EACnD,CAAC;QACF,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3B,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,kBAAkB,EAAE,CAAC;QACvC,IAAI,MAAM,eAAe,CAAC,CAAC,GAAG,MAAM,GAAG,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC;YAAE,MAAK;IACpE,CAAC;IACD,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;AAC3B,CAAC;AAED,gEAAgE;AAChE,MAAM,UAAU,mBAAmB,CAAC,GAAW;IAC9C,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;QAC3B,MAAM,CAAC,IAAI,GAAG,EAAE,CAAA;QAChB,OAAO,MAAM,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IAC7C,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IAC/B,CAAC;AACF,CAAC","sourcesContent":["import { escapeRegExp } from 'hanzio'\nimport { throughCrawlSnapshot } from '../crawl-snapshot.js'\n\nconst FETCH_TIMEOUT_MS = 10_000\nconst MAX_SITEMAP_FILES = 20\nconst MAX_URLS = 20_000\n\n/** Conventional sitemap locations, tried in order when robots.txt names none. */\nconst SITEMAP_ROOT_PATHS = [\n\t'/sitemap.xml',\n\t'/sitemap_index.xml',\n\t'/wp-sitemap.xml'\n]\n\nconst LOC_RE = /<loc>\\s*([^<]+?)\\s*<\\/loc>/gi\nconst URL_ENTRY_RE = /<url>([\\s\\S]*?)<\\/url>/gi\nconst SITEMAP_ENTRY_RE = /<sitemap>([\\s\\S]*?)<\\/sitemap>/gi\nconst ENTRY_LOC_RE = /<loc>\\s*([^<]+?)\\s*<\\/loc>/i\nconst LASTMOD_RE = /<lastmod>\\s*([^<]+?)\\s*<\\/lastmod>/i\nconst ROBOTS_SITEMAP_RE = /^[^\\S\\r\\n]*sitemap[^\\S\\r\\n]*:[^\\S\\r\\n]*(\\S+)/gim\n\nconst XML_ENTITIES = new Map([\n\t['&amp;', '&'],\n\t['&lt;', '<'],\n\t['&gt;', '>'],\n\t['&quot;', '\"'],\n\t['&apos;', \"'\"],\n\t['&#39;', \"'\"]\n])\n\n/** One pass, so `&amp;lt;` decodes to `&lt;` and not to `<`. */\nfunction decodeXmlEntities(value: string) {\n\treturn value.replace(\n\t\t/&(?:amp|lt|gt|quot|apos|#39);/g,\n\t\t(entity) => XML_ENTITIES.get(entity) ?? entity\n\t)\n}\n\n/** Parses one sitemap document into page entries and nested sitemap URLs. */\nexport function parseSitemapXml(xml: string) {\n\tconst pages: { url: string; lastmod: string | null }[] = []\n\tconst nested: string[] = []\n\n\tfor (const match of xml.matchAll(URL_ENTRY_RE)) {\n\t\tconst entry = match[1] ?? ''\n\t\tconst loc = entry.match(ENTRY_LOC_RE)?.[1]\n\t\tif (!loc) continue\n\t\tpages.push({\n\t\t\turl: decodeXmlEntities(loc),\n\t\t\tlastmod: entry.match(LASTMOD_RE)?.[1]?.trim() ?? null\n\t\t})\n\t}\n\tfor (const match of xml.matchAll(SITEMAP_ENTRY_RE)) {\n\t\tconst loc = match[1]?.match(ENTRY_LOC_RE)?.[1]\n\t\tif (loc) nested.push(decodeXmlEntities(loc))\n\t}\n\t// Plain `<loc>` lists without `<url>` wrappers (rare, but seen in the wild).\n\tif (pages.length === 0 && nested.length === 0) {\n\t\tfor (const match of xml.matchAll(LOC_RE)) {\n\t\t\tif (match[1])\n\t\t\t\tpages.push({ url: decodeXmlEntities(match[1]), lastmod: null })\n\t\t}\n\t}\n\treturn { pages, nested }\n}\n\nfunction fetchText(url: string, accept: string) {\n\treturn throughCrawlSnapshot(`text ${accept} ${url}`, () =>\n\t\tfetchTextLive(url, accept)\n\t)\n}\n\nasync function fetchTextLive(url: string, accept: string) {\n\ttry {\n\t\tconst response = await fetch(url, {\n\t\t\tsignal: AbortSignal.timeout(FETCH_TIMEOUT_MS),\n\t\t\theaders: { accept }\n\t\t})\n\t\tif (!response.ok) return null\n\t\tconst type = response.headers.get('content-type') ?? ''\n\t\t// A CMS that renders its 404 page for a missing file answers 200 with HTML.\n\t\tif (type.includes('text/html')) return null\n\t\treturn await response.text()\n\t} catch {\n\t\treturn null\n\t}\n}\n\nfunction originOf(websiteUrl: string) {\n\ttry {\n\t\treturn new URL(websiteUrl).origin\n\t} catch {\n\t\treturn null\n\t}\n}\n\n/**\n * `/robots.txt`, read once: its `Sitemap:` lines, absolute-ised (many CMSs\n * publish their sitemap somewhere other than `/sitemap.xml` and only say so\n * here), and its rules. An HTML body or any non-200 means \"no robots.txt\";\n * never throws.\n */\nexport async function fetchRobots(websiteUrl: string) {\n\tconst sitemaps: string[] = []\n\tconst origin = originOf(websiteUrl)\n\tif (!origin) return { sitemaps, rules: [] }\n\tconst robotsUrl = `${origin}/robots.txt`\n\tconst body = await fetchText(robotsUrl, 'text/plain,*/*;q=0.8')\n\tif (!body) return { sitemaps, rules: [] }\n\n\tfor (const match of body.matchAll(ROBOTS_SITEMAP_RE)) {\n\t\tconst value = match[1]\n\t\tif (!value) continue\n\t\ttry {\n\t\t\tconst absolute = new URL(value, robotsUrl)\n\t\t\t// Only what we could actually GET - `ftp:` and friends are noise.\n\t\t\tif (absolute.protocol !== 'http:' && absolute.protocol !== 'https:')\n\t\t\t\tcontinue\n\t\t\tconst href = absolute.toString()\n\t\t\tif (!sitemaps.includes(href)) sitemaps.push(href)\n\t\t} catch {\n\t\t\t// A malformed Sitemap: line must not cost us the other lines.\n\t\t}\n\t}\n\treturn { sitemaps, rules: parseRobotsRules(body) }\n}\n\nexport type RobotsRule = { allow: boolean; pattern: string }\n\n/**\n * The `Allow`/`Disallow` lines of the `User-agent: *` groups. We read pages\n * with a browser user agent, so a group naming a specific crawler is not ours;\n * the wildcard group is what the site asks of everyone. An empty `Disallow:`\n * allows everything and adds no rule.\n */\nexport function parseRobotsRules(body: string) {\n\tconst rules: RobotsRule[] = []\n\tlet agents: string[] = []\n\tlet inRules = false\n\n\tfor (const raw of body.split(/\\r?\\n/)) {\n\t\tconst match = /^\\s*([a-z-]+)\\s*:\\s*(.*?)\\s*$/i.exec(raw.replace(/#.*$/, ''))\n\t\tif (!match) continue\n\t\tconst field = (match[1] ?? '').toLowerCase()\n\t\tconst value = match[2] ?? ''\n\n\t\tif (field === 'user-agent') {\n\t\t\t// A user-agent line after rules opens a new group.\n\t\t\tif (inRules) agents = []\n\t\t\tinRules = false\n\t\t\tagents.push(value.toLowerCase())\n\t\t\tcontinue\n\t\t}\n\t\tif (field !== 'allow' && field !== 'disallow') continue\n\n\t\tinRules = true\n\t\tif (agents.includes('*') && value !== '')\n\t\t\trules.push({ allow: field === 'allow', pattern: value })\n\t}\n\treturn rules\n}\n\n/** Compiled once per pattern: the crawl asks for every link of every page. */\nconst robotsPatternCache = new Map<string, RegExp>()\n\nfunction robotsPatternRegex(pattern: string) {\n\tconst cached = robotsPatternCache.get(pattern)\n\tif (cached) return cached\n\n\tconst anchored = pattern.endsWith('$')\n\tconst body = (anchored ? pattern.slice(0, -1) : pattern)\n\t\t.split('*')\n\t\t.map((part) => escapeRegExp(part))\n\t\t.join('.*')\n\tconst regex = new RegExp(`^${body}${anchored ? '$' : ''}`)\n\t// Every site's patterns land here; a long-lived worker starts over rather than grow.\n\tif (robotsPatternCache.size >= 5_000) robotsPatternCache.clear()\n\trobotsPatternCache.set(pattern, regex)\n\treturn regex\n}\n\n/**\n * Whether robots.txt lets us read this URL: the longest matching rule wins and\n * `Allow` wins a tie, as Google reads it. Shopify disallows `/cart/`, and the\n * crawler had fetched `/cart/add?id=…&quantity=1` - a GET that fills a cart\n * (Good Habit audit §3.5).\n */\nexport function robotsAllows(rules: RobotsRule[], url: string) {\n\tlet path: string\n\ttry {\n\t\tconst parsed = new URL(url)\n\t\tpath = parsed.pathname + parsed.search\n\t} catch {\n\t\treturn true\n\t}\n\n\tlet best: RobotsRule | null = null\n\tfor (const rule of rules) {\n\t\tif (!robotsPatternRegex(rule.pattern).test(path)) continue\n\t\tif (!best || rule.pattern.length > best.pattern.length) best = rule\n\t\telse if (rule.pattern.length === best.pattern.length && rule.allow)\n\t\t\tbest = rule\n\t}\n\treturn best?.allow ?? true\n}\n\n/**\n * Reads a set of sitemap roots (following index nesting) into `found`.\n * Returns true when at least one of them yielded page URLs, so the caller can\n * stop trying further roots.\n */\nasync function readSitemapTree(\n\troots: string[],\n\tfound: Map<string, { url: string; lastmod: string | null }>,\n\tseen: Set<string>\n) {\n\tlet readable = false\n\tlet level = [...roots]\n\n\t// One level of the index at a time, its files read side by side: a shop\n\t// with a sitemap per locale and type read up to 20 files one after another,\n\t// on the path to the first page read. Results are applied in listed order,\n\t// so which copy of a URL is kept does not depend on network timing.\n\twhile (level.length > 0 && found.size < MAX_URLS) {\n\t\tconst batch: string[] = []\n\t\tfor (const sitemapUrl of level) {\n\t\t\tif (seen.has(sitemapUrl) || seen.size >= MAX_SITEMAP_FILES) continue\n\t\t\tseen.add(sitemapUrl)\n\t\t\tbatch.push(sitemapUrl)\n\t\t}\n\t\tconst bodies = await Promise.all(\n\t\t\tbatch.map((sitemapUrl) =>\n\t\t\t\tfetchText(sitemapUrl, 'application/xml,text/xml;q=0.9,*/*;q=0.8')\n\t\t\t)\n\t\t)\n\t\tconst next: string[] = []\n\t\tfor (const xml of bodies) {\n\t\t\tif (!xml) continue\n\t\t\tconst { pages, nested } = parseSitemapXml(xml)\n\t\t\t// An empty document is not a sitemap we can use - let the next root try.\n\t\t\tif (pages.length === 0 && nested.length === 0) continue\n\t\t\tif (pages.length > 0) readable = true\n\t\t\tnext.push(...nested)\n\t\t\tfor (const page of pages) {\n\t\t\t\tif (found.size >= MAX_URLS) break\n\t\t\t\t// Keyed on the normalised form so `/kontakt` and `/kontakt/` listed in\n\t\t\t\t// two sitemap files do not become two fetches.\n\t\t\t\tconst key = normalizeSitemapUrl(page.url)\n\t\t\t\tif (!found.has(key)) found.set(key, page)\n\t\t\t}\n\t\t}\n\t\tlevel = next\n\t}\n\treturn readable\n}\n\n/**\n * Every URL the site's sitemaps list, robots-declared sitemaps (`fetchRobots`)\n * first and the conventional roots as a fallback. Returns `[]` when the site has no usable\n * sitemap - a fifth of real customer sites have none, which is normal and not\n * an error: link-following discovery is the source, sitemaps a supplement.\n * Never throws.\n */\nexport async function discoverSitemapUrls(\n\twebsiteUrl: string,\n\trobotsSitemaps: string[]\n) {\n\tconst origin = originOf(websiteUrl)\n\tif (!origin) return []\n\tconst found = new Map<string, { url: string; lastmod: string | null }>()\n\t// Shared across roots so a broken sitemap index cannot spend the file budget\n\t// several times over.\n\tconst seen = new Set<string>()\n\n\t// robots.txt may declare several sitemaps (WordPress splits per post type),\n\t// so all of them are read together - only the conventional roots compete.\n\tif (\n\t\trobotsSitemaps.length > 0 &&\n\t\t(await readSitemapTree(robotsSitemaps, found, seen))\n\t) {\n\t\treturn [...found.values()]\n\t}\n\tfor (const path of SITEMAP_ROOT_PATHS) {\n\t\tif (await readSitemapTree([`${origin}${path}`], found, seen)) break\n\t}\n\treturn [...found.values()]\n}\n\n/** One key per page whatever the trailing slash or fragment. */\nexport function normalizeSitemapUrl(url: string) {\n\ttry {\n\t\tconst parsed = new URL(url)\n\t\tparsed.hash = ''\n\t\treturn parsed.toString().replace(/\\/+$/, '')\n\t} catch {\n\t\treturn url.replace(/\\/+$/, '')\n\t}\n}\n"]}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/sitemap/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,QAAQ,CAAA;AACpE,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAClE,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAA;AAC3D,OAAO,EAAc,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC1D,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAA;AACtD,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAIrD,MAAM,gBAAgB,GAAG,MAAM,CAAA;AAC/B,MAAM,iBAAiB,GAAG,EAAE,CAAA;AAC5B,MAAM,QAAQ,GAAG,MAAM,CAAA;AACvB,2EAA2E;AAC3E,MAAM,gBAAgB,GAAG,GAAG,GAAG,IAAI,CAAA;AACnC;;;GAGG;AACH,MAAM,iBAAiB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAA;AAW1C,iFAAiF;AACjF,MAAM,kBAAkB,GAAG;IAC1B,cAAc;IACd,oBAAoB;IACpB,iBAAiB;CACjB,CAAA;AAED,MAAM,MAAM,GAAG,8BAA8B,CAAA;AAC7C,MAAM,YAAY,GAAG,0BAA0B,CAAA;AAC/C,MAAM,gBAAgB,GAAG,kCAAkC,CAAA;AAC3D,MAAM,YAAY,GAAG,6BAA6B,CAAA;AAClD,MAAM,UAAU,GAAG,qCAAqC,CAAA;AACxD,MAAM,iBAAiB,GAAG,iDAAiD,CAAA;AAE3E,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC;IAC5B,CAAC,OAAO,EAAE,GAAG,CAAC;IACd,CAAC,MAAM,EAAE,GAAG,CAAC;IACb,CAAC,MAAM,EAAE,GAAG,CAAC;IACb,CAAC,QAAQ,EAAE,GAAG,CAAC;IACf,CAAC,QAAQ,EAAE,GAAG,CAAC;IACf,CAAC,OAAO,EAAE,GAAG,CAAC;CACd,CAAC,CAAA;AAEF,gEAAgE;AAChE,SAAS,iBAAiB,CAAC,KAAa;IACvC,OAAO,KAAK,CAAC,OAAO,CACnB,gCAAgC,EAChC,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,MAAM,CAC9C,CAAA;AACF,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,eAAe,CAAC,GAAW;IAC1C,MAAM,KAAK,GAA8C,EAAE,CAAA;IAC3D,MAAM,MAAM,GAAa,EAAE,CAAA;IAE3B,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;QAChD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;QAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAC1C,IAAI,CAAC,GAAG;YAAE,SAAQ;QAClB,KAAK,CAAC,IAAI,CAAC;YACV,GAAG,EAAE,iBAAiB,CAAC,GAAG,CAAC;YAC3B,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,IAAI;SACrD,CAAC,CAAA;IACH,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACpD,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAC9C,IAAI,GAAG;YAAE,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAA;IAC7C,CAAC;IACD,6EAA6E;IAC7E,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/C,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1C,IAAI,KAAK,CAAC,CAAC,CAAC;gBACX,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;QACjE,CAAC;IACF,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAA;AACzB,CAAC;AAID,SAAS,SAAS,CAAC,GAAW,EAAE,OAAoB;IACnD,OAAO,oBAAoB,CAAC,QAAQ,OAAO,CAAC,MAAM,IAAI,GAAG,EAAE,EAAE,GAAG,EAAE,CACjE,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAC3B,CAAA;AACF,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,GAAW,EAAE,OAAoB;IAC7D,IAAI,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE;YACzC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,gBAAgB,CAAC;YAC7C,OAAO,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,kBAAkB,EAAE;SACrE,CAAC,CAAA;QACF,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAA;QACvD,4EAA4E;QAC5E,IAAI,CAAC,QAAQ,CAAC,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YAChD,KAAK,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;YAC5C,OAAO,IAAI,CAAA;QACZ,CAAC;QACD,OAAO,UAAU,CAAC,CAAC,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;IACtE,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC;AAED,MAAM,cAAc,GAAG;IACtB,MAAM,EAAE,sBAAsB;IAC9B,QAAQ,EAAE,gBAAgB;CAC1B,CAAA;AACD,MAAM,eAAe,GAAG;IACvB,MAAM,EAAE,0CAA0C;IAClD,QAAQ,EAAE,iBAAiB;CAC3B,CAAA;AAED,SAAS,QAAQ,CAAC,UAAkB;IACnC,IAAI,CAAC;QACJ,OAAO,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,MAAM,CAAA;IAClC,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAChC,UAAkB,EAClB,UAA0B,EAAE;IAE5B,MAAM,QAAQ,GAAa,EAAE,CAAA;IAC7B,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAA;IACnC,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,EAAE,CAAA;IAC3C,MAAM,SAAS,GAAG,GAAG,MAAM,aAAa,CAAA;IACxC,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,SAAS,EAAE;QACvC,GAAG,cAAc;QACjB,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,YAAY;KACpC,CAAC,CAAA;IACF,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,EAAE,CAAA;IAEzC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;QACtD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACtB,IAAI,CAAC,KAAK;YAAE,SAAQ;QACpB,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;YAC1C,kEAAkE;YAClE,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC;gBAAE,SAAQ;YAC9D,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAA;QACnC,CAAC;QAAC,MAAM,CAAC;YACR,8DAA8D;QAC/D,CAAC;IACF,CAAC;IACD,OAAO;QACN,QAAQ,EAAE,eAAe,CAAC,QAAQ,CAAC;QACnC,KAAK,EAAE,gBAAgB,CAAC,IAAI,CAAC;KAC7B,CAAA;AACF,CAAC;AAID;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC5C,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC,CAAA;IAChD,OAAO;QACN,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;QACrD,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;KACzD,CAAA;AACF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,KAAmB,EAAE,GAAW;IAC5D,OAAO,iBAAiB,CACvB;QACC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC;QACrE,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC;KACzE,EACD,GAAG,CACH,CAAA;AACF,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,eAAe,CAC7B,KAAe,EACf,KAA2D,EAC3D,IAAiB,EACjB,KAAY;IAEZ,IAAI,QAAQ,GAAG,KAAK,CAAA;IACpB,IAAI,KAAK,GAAG,CAAC,GAAG,KAAK,CAAC,CAAA;IAEtB,wEAAwE;IACxE,4EAA4E;IAC5E,2EAA2E;IAC3E,oEAAoE;IACpE,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,GAAG,QAAQ,EAAE,CAAC;QAClD,MAAM,KAAK,GAAa,EAAE,CAAA;QAC1B,KAAK,MAAM,UAAU,IAAI,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,IAAI,IAAI,iBAAiB;gBAAE,SAAQ;YACpE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;YACpB,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;QACvB,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CACxB,SAAS,CAAC,UAAU,EAAE,EAAE,GAAG,eAAe,EAAE,KAAK,EAAE,CAAC,CACpD,CACD,CAAA;QACD,MAAM,IAAI,GAAa,EAAE,CAAA;QACzB,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YAC1B,IAAI,CAAC,GAAG;gBAAE,SAAQ;YAClB,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,eAAe,CAAC,GAAG,CAAC,CAAA;YAC9C,yEAAyE;YACzE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAQ;YACvD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;gBAAE,QAAQ,GAAG,IAAI,CAAA;YACrC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAA;YACpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBAC1B,IAAI,KAAK,CAAC,IAAI,IAAI,QAAQ;oBAAE,MAAK;gBACjC,uEAAuE;gBACvE,+CAA+C;gBAC/C,MAAM,GAAG,GAAG,mBAAmB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACzC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC;oBAAE,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;YAC1C,CAAC;QACF,CAAC;QACD,KAAK,GAAG,IAAI,CAAA;IACb,CAAC;IACD,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACxC,UAAkB,EAClB,cAAwB,EACxB,UAA0B,EAAE;IAE5B,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAA;IACnC,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,CAAA;IACtB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,YAAY,CAAA;IAC3C,MAAM,KAAK,GAAG,IAAI,GAAG,EAAmD,CAAA;IACxE,6EAA6E;IAC7E,sBAAsB;IACtB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAE9B,4EAA4E;IAC5E,0EAA0E;IAC1E,IACC,cAAc,CAAC,MAAM,GAAG,CAAC;QACzB,CAAC,MAAM,eAAe,CAAC,cAAc,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,EAC1D,CAAC;QACF,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3B,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,kBAAkB,EAAE,CAAC;QACvC,IAAI,MAAM,eAAe,CAAC,CAAC,GAAG,MAAM,GAAG,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC;YAAE,MAAK;IAC3E,CAAC;IACD,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;AAC3B,CAAC;AAED,gEAAgE;AAChE,MAAM,UAAU,mBAAmB,CAAC,GAAW;IAC9C,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;QAC3B,MAAM,CAAC,IAAI,GAAG,EAAE,CAAA;QAChB,OAAO,iBAAiB,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAA;IAC5C,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,iBAAiB,CAAC,GAAG,CAAC,CAAA;IAC9B,CAAC;AACF,CAAC","sourcesContent":["import { getUniqueValues, isOneOf, trimTrailingSlash } from 'hanzio'\nimport { isAllowedByRobots, parseRobotsTxt } from 'hanzio/sitemap'\nimport { throughCrawlSnapshot } from '../crawl-snapshot.js'\nimport { type Fetch, guardedFetch } from '../net-guard.js'\nimport { decodeText, readBody } from '../read-body.js'\nimport { BROWSER_USER_AGENT } from '../user-agent.js'\n\nexport type { Fetch }\n\nconst FETCH_TIMEOUT_MS = 10_000\nconst MAX_SITEMAP_FILES = 20\nconst MAX_URLS = 20_000\n/** Google reads the first 500 KiB of a robots.txt and ignores the rest. */\nconst MAX_ROBOTS_BYTES = 500 * 1024\n/**\n * Read cap per sitemap file. 20,000 URLs fit in a few MB; a longer file is\n * read up to here and its complete entries used.\n */\nconst MAX_SITEMAP_BYTES = 10 * 1024 * 1024\n\nexport type SitemapOptions = {\n\t/**\n\t * The `fetch` robots.txt and sitemaps go through. Default\n\t * {@link guardedFetch} with the default guard (public addresses only, every\n\t * redirect checked); pass `converter.guardedFetch` to use the converter's.\n\t */\n\tfetch?: Fetch | undefined\n}\n\n/** Conventional sitemap locations, tried in order when robots.txt names none. */\nconst SITEMAP_ROOT_PATHS = [\n\t'/sitemap.xml',\n\t'/sitemap_index.xml',\n\t'/wp-sitemap.xml'\n]\n\nconst LOC_RE = /<loc>\\s*([^<]+?)\\s*<\\/loc>/gi\nconst URL_ENTRY_RE = /<url>([\\s\\S]*?)<\\/url>/gi\nconst SITEMAP_ENTRY_RE = /<sitemap>([\\s\\S]*?)<\\/sitemap>/gi\nconst ENTRY_LOC_RE = /<loc>\\s*([^<]+?)\\s*<\\/loc>/i\nconst LASTMOD_RE = /<lastmod>\\s*([^<]+?)\\s*<\\/lastmod>/i\nconst ROBOTS_SITEMAP_RE = /^[^\\S\\r\\n]*sitemap[^\\S\\r\\n]*:[^\\S\\r\\n]*(\\S+)/gim\n\nconst XML_ENTITIES = new Map([\n\t['&amp;', '&'],\n\t['&lt;', '<'],\n\t['&gt;', '>'],\n\t['&quot;', '\"'],\n\t['&apos;', \"'\"],\n\t['&#39;', \"'\"]\n])\n\n/** One pass, so `&amp;lt;` decodes to `&lt;` and not to `<`. */\nfunction decodeXmlEntities(value: string) {\n\treturn value.replace(\n\t\t/&(?:amp|lt|gt|quot|apos|#39);/g,\n\t\t(entity) => XML_ENTITIES.get(entity) ?? entity\n\t)\n}\n\n/** Parses one sitemap document into page entries and nested sitemap URLs. */\nexport function parseSitemapXml(xml: string) {\n\tconst pages: { url: string; lastmod: string | null }[] = []\n\tconst nested: string[] = []\n\n\tfor (const match of xml.matchAll(URL_ENTRY_RE)) {\n\t\tconst entry = match[1] ?? ''\n\t\tconst loc = entry.match(ENTRY_LOC_RE)?.[1]\n\t\tif (!loc) continue\n\t\tpages.push({\n\t\t\turl: decodeXmlEntities(loc),\n\t\t\tlastmod: entry.match(LASTMOD_RE)?.[1]?.trim() ?? null\n\t\t})\n\t}\n\tfor (const match of xml.matchAll(SITEMAP_ENTRY_RE)) {\n\t\tconst loc = match[1]?.match(ENTRY_LOC_RE)?.[1]\n\t\tif (loc) nested.push(decodeXmlEntities(loc))\n\t}\n\t// Plain `<loc>` lists without `<url>` wrappers (rare, but seen in the wild).\n\tif (pages.length === 0 && nested.length === 0) {\n\t\tfor (const match of xml.matchAll(LOC_RE)) {\n\t\t\tif (match[1])\n\t\t\t\tpages.push({ url: decodeXmlEntities(match[1]), lastmod: null })\n\t\t}\n\t}\n\treturn { pages, nested }\n}\n\ntype TextRequest = { accept: string; maxBytes: number; fetch: Fetch }\n\nfunction fetchText(url: string, request: TextRequest) {\n\treturn throughCrawlSnapshot(`text ${request.accept} ${url}`, () =>\n\t\tfetchTextLive(url, request)\n\t)\n}\n\nasync function fetchTextLive(url: string, request: TextRequest) {\n\ttry {\n\t\tconst response = await request.fetch(url, {\n\t\t\tsignal: AbortSignal.timeout(FETCH_TIMEOUT_MS),\n\t\t\theaders: { accept: request.accept, 'user-agent': BROWSER_USER_AGENT }\n\t\t})\n\t\tconst type = response.headers.get('content-type') ?? ''\n\t\t// A CMS that renders its 404 page for a missing file answers 200 with HTML.\n\t\tif (!response.ok || type.includes('text/html')) {\n\t\t\tvoid response.body?.cancel().catch(() => {})\n\t\t\treturn null\n\t\t}\n\t\treturn decodeText((await readBody(response, request.maxBytes)).bytes)\n\t} catch {\n\t\treturn null\n\t}\n}\n\nconst ROBOTS_REQUEST = {\n\taccept: 'text/plain,*/*;q=0.8',\n\tmaxBytes: MAX_ROBOTS_BYTES\n}\nconst SITEMAP_REQUEST = {\n\taccept: 'application/xml,text/xml;q=0.9,*/*;q=0.8',\n\tmaxBytes: MAX_SITEMAP_BYTES\n}\n\nfunction originOf(websiteUrl: string) {\n\ttry {\n\t\treturn new URL(websiteUrl).origin\n\t} catch {\n\t\treturn null\n\t}\n}\n\n/**\n * `/robots.txt`, read once: its `Sitemap:` lines, absolute-ised (many CMSs\n * publish their sitemap somewhere other than `/sitemap.xml` and only say so\n * here), and its rules. An HTML body or any non-200 means \"no robots.txt\";\n * never throws.\n */\nexport async function fetchRobots(\n\twebsiteUrl: string,\n\toptions: SitemapOptions = {}\n) {\n\tconst sitemaps: string[] = []\n\tconst origin = originOf(websiteUrl)\n\tif (!origin) return { sitemaps, rules: [] }\n\tconst robotsUrl = `${origin}/robots.txt`\n\tconst body = await fetchText(robotsUrl, {\n\t\t...ROBOTS_REQUEST,\n\t\tfetch: options.fetch ?? guardedFetch\n\t})\n\tif (!body) return { sitemaps, rules: [] }\n\n\tfor (const match of body.matchAll(ROBOTS_SITEMAP_RE)) {\n\t\tconst value = match[1]\n\t\tif (!value) continue\n\t\ttry {\n\t\t\tconst absolute = new URL(value, robotsUrl)\n\t\t\t// Only what we could actually GET - `ftp:` and friends are noise.\n\t\t\tif (!isOneOf(['http:', 'https:'], absolute.protocol)) continue\n\t\t\tsitemaps.push(absolute.toString())\n\t\t} catch {\n\t\t\t// A malformed Sitemap: line must not cost us the other lines.\n\t\t}\n\t}\n\treturn {\n\t\tsitemaps: getUniqueValues(sitemaps),\n\t\trules: parseRobotsRules(body)\n\t}\n}\n\nexport type RobotsRule = { allow: boolean; pattern: string }\n\n/**\n * The `Allow`/`Disallow` lines of the `User-agent: *` groups. We read pages\n * with a browser user agent, so a group naming a specific crawler is not ours;\n * the wildcard group is what the site asks of everyone. An empty `Disallow:`\n * allows everything and adds no rule.\n */\nexport function parseRobotsRules(body: string): RobotsRule[] {\n\tconst { allow, disallow } = parseRobotsTxt(body)\n\treturn [\n\t\t...allow.map((pattern) => ({ allow: true, pattern })),\n\t\t...disallow.map((pattern) => ({ allow: false, pattern }))\n\t]\n}\n\n/**\n * Whether robots.txt lets us read this URL: the longest matching rule wins and\n * `Allow` wins a tie, as Google reads it. Shopify disallows `/cart/`, and the\n * crawler had fetched `/cart/add?id=…&quantity=1` - a GET that fills a cart.\n */\nexport function robotsAllows(rules: RobotsRule[], url: string) {\n\treturn isAllowedByRobots(\n\t\t{\n\t\t\tallow: rules.filter((rule) => rule.allow).map((rule) => rule.pattern),\n\t\t\tdisallow: rules.filter((rule) => !rule.allow).map((rule) => rule.pattern)\n\t\t},\n\t\turl\n\t)\n}\n\n/**\n * Reads a set of sitemap roots (following index nesting) into `found`.\n * Returns true when at least one of them yielded page URLs, so the caller can\n * stop trying further roots.\n */\nasync function readSitemapTree(\n\troots: string[],\n\tfound: Map<string, { url: string; lastmod: string | null }>,\n\tseen: Set<string>,\n\tfetch: Fetch\n) {\n\tlet readable = false\n\tlet level = [...roots]\n\n\t// One level of the index at a time, its files read side by side: a shop\n\t// with a sitemap per locale and type read up to 20 files one after another,\n\t// on the path to the first page read. Results are applied in listed order,\n\t// so which copy of a URL is kept does not depend on network timing.\n\twhile (level.length > 0 && found.size < MAX_URLS) {\n\t\tconst batch: string[] = []\n\t\tfor (const sitemapUrl of level) {\n\t\t\tif (seen.has(sitemapUrl) || seen.size >= MAX_SITEMAP_FILES) continue\n\t\t\tseen.add(sitemapUrl)\n\t\t\tbatch.push(sitemapUrl)\n\t\t}\n\t\tconst bodies = await Promise.all(\n\t\t\tbatch.map((sitemapUrl) =>\n\t\t\t\tfetchText(sitemapUrl, { ...SITEMAP_REQUEST, fetch })\n\t\t\t)\n\t\t)\n\t\tconst next: string[] = []\n\t\tfor (const xml of bodies) {\n\t\t\tif (!xml) continue\n\t\t\tconst { pages, nested } = parseSitemapXml(xml)\n\t\t\t// An empty document is not a sitemap we can use - let the next root try.\n\t\t\tif (pages.length === 0 && nested.length === 0) continue\n\t\t\tif (pages.length > 0) readable = true\n\t\t\tnext.push(...nested)\n\t\t\tfor (const page of pages) {\n\t\t\t\tif (found.size >= MAX_URLS) break\n\t\t\t\t// Keyed on the normalised form so `/kontakt` and `/kontakt/` listed in\n\t\t\t\t// two sitemap files do not become two fetches.\n\t\t\t\tconst key = normalizeSitemapUrl(page.url)\n\t\t\t\tif (!found.has(key)) found.set(key, page)\n\t\t\t}\n\t\t}\n\t\tlevel = next\n\t}\n\treturn readable\n}\n\n/**\n * Every URL the site's sitemaps list, robots-declared sitemaps (`fetchRobots`)\n * first and the conventional roots as a fallback. Returns `[]` when the site has no usable\n * sitemap - a fifth of real customer sites have none, which is normal and not\n * an error: link-following discovery is the source, sitemaps a supplement.\n * Never throws.\n */\nexport async function discoverSitemapUrls(\n\twebsiteUrl: string,\n\trobotsSitemaps: string[],\n\toptions: SitemapOptions = {}\n) {\n\tconst origin = originOf(websiteUrl)\n\tif (!origin) return []\n\tconst fetch = options.fetch ?? guardedFetch\n\tconst found = new Map<string, { url: string; lastmod: string | null }>()\n\t// Shared across roots so a broken sitemap index cannot spend the file budget\n\t// several times over.\n\tconst seen = new Set<string>()\n\n\t// robots.txt may declare several sitemaps (WordPress splits per post type),\n\t// so all of them are read together - only the conventional roots compete.\n\tif (\n\t\trobotsSitemaps.length > 0 &&\n\t\t(await readSitemapTree(robotsSitemaps, found, seen, fetch))\n\t) {\n\t\treturn [...found.values()]\n\t}\n\tfor (const path of SITEMAP_ROOT_PATHS) {\n\t\tif (await readSitemapTree([`${origin}${path}`], found, seen, fetch)) break\n\t}\n\treturn [...found.values()]\n}\n\n/** One key per page whatever the trailing slash or fragment. */\nexport function normalizeSitemapUrl(url: string) {\n\ttry {\n\t\tconst parsed = new URL(url)\n\t\tparsed.hash = ''\n\t\treturn trimTrailingSlash(parsed.toString())\n\t} catch {\n\t\treturn trimTrailingSlash(url)\n\t}\n}\n"]}

package/dist/testing/index.d.ts

@@ -0,0 +1,16 @@
+import type { FetchedPage, FetchPageInput } from '../fetch-page.js';
+/** A complete `FetchedPage`: an `ok` HTML read of `url` with nothing on it, plus `page`. */
+export declare function fakePage(page: Partial<FetchedPage> & {
+    url: string;
+}): FetchedPage;
+/**
+ * A site in memory with the converter's `fetchPageAsMarkdown`: the pages
+ * given by URL, `gone` (404) for any other, `unchanged` (304) when the input's
+ * `etag`, `lastModified` or `htmlHash` matches the page's. `calls` lists every
+ * input in order.
+ */
+export declare function fakeSite(pages: Record<string, Omit<Partial<FetchedPage>, 'url'>>): {
+    fetchPageAsMarkdown: (input: FetchPageInput) => Promise<FetchedPage>;
+    calls: FetchPageInput[];
+};
+//# sourceMappingURL=index.d.ts.map

package/dist/testing/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/testing/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AAGnE,4FAA4F;AAC5F,wBAAgB,QAAQ,CACvB,IAAI,EAAE,OAAO,CAAC,WAAW,CAAC,GAAG;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,GAC1C,WAAW,CAwBb;AAED;;;;;GAKG;AACH,wBAAgB,QAAQ,CACvB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,KAAK,CAAC,CAAC;iCAGd,cAAc;;EAiBxD"}

package/dist/testing/index.js

@@ -0,0 +1,54 @@
+import { asUnchanged, isUnchangedFor } from '../unchanged-page.js';
+/** A complete `FetchedPage`: an `ok` HTML read of `url` with nothing on it, plus `page`. */
+export function fakePage(page) {
+    return {
+        finalUrl: page.url,
+        status: 200,
+        outcome: 'ok',
+        source: 'workers-ai',
+        contentType: 'text/html; charset=utf-8',
+        etag: null,
+        lastModified: null,
+        markdown: '',
+        title: null,
+        lang: null,
+        canonical: null,
+        alternates: [],
+        alternateLanguages: {},
+        metaDescription: null,
+        jsonLd: [],
+        links: [],
+        images: [],
+        durationMs: 0,
+        captureIssue: null,
+        htmlHash: null,
+        ...page
+    };
+}
+/**
+ * A site in memory with the converter's `fetchPageAsMarkdown`: the pages
+ * given by URL, `gone` (404) for any other, `unchanged` (304) when the input's
+ * `etag`, `lastModified` or `htmlHash` matches the page's. `calls` lists every
+ * input in order.
+ */
+export function fakeSite(pages) {
+    const calls = [];
+    async function fetchPageAsMarkdown(input) {
+        calls.push(input);
+        const known = pages[input.url];
+        if (!known) {
+            return fakePage({
+                url: input.url,
+                status: 404,
+                outcome: 'gone',
+                source: null
+            });
+        }
+        const page = fakePage({ ...known, url: input.url });
+        return isUnchangedFor(page, input)
+            ? { ...asUnchanged(page), status: 304 }
+            : page;
+    }
+    return { fetchPageAsMarkdown, calls };
+}
+//# sourceMappingURL=index.js.map

package/dist/testing/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/testing/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AAElE,4FAA4F;AAC5F,MAAM,UAAU,QAAQ,CACvB,IAA4C;IAE5C,OAAO;QACN,QAAQ,EAAE,IAAI,CAAC,GAAG;QAClB,MAAM,EAAE,GAAG;QACX,OAAO,EAAE,IAAI;QACb,MAAM,EAAE,YAAY;QACpB,WAAW,EAAE,0BAA0B;QACvC,IAAI,EAAE,IAAI;QACV,YAAY,EAAE,IAAI;QAClB,QAAQ,EAAE,EAAE;QACZ,KAAK,EAAE,IAAI;QACX,IAAI,EAAE,IAAI;QACV,SAAS,EAAE,IAAI;QACf,UAAU,EAAE,EAAE;QACd,kBAAkB,EAAE,EAAE;QACtB,eAAe,EAAE,IAAI;QACrB,MAAM,EAAE,EAAE;QACV,KAAK,EAAE,EAAE;QACT,MAAM,EAAE,EAAE;QACV,UAAU,EAAE,CAAC;QACb,YAAY,EAAE,IAAI;QAClB,QAAQ,EAAE,IAAI;QACd,GAAG,IAAI;KACP,CAAA;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,QAAQ,CACvB,KAAwD;IAExD,MAAM,KAAK,GAAqB,EAAE,CAAA;IAClC,KAAK,UAAU,mBAAmB,CAAC,KAAqB;QACvD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACjB,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAC9B,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,OAAO,QAAQ,CAAC;gBACf,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,MAAM,EAAE,GAAG;gBACX,OAAO,EAAE,MAAM;gBACf,MAAM,EAAE,IAAI;aACZ,CAAC,CAAA;QACH,CAAC;QACD,MAAM,IAAI,GAAG,QAAQ,CAAC,EAAE,GAAG,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,CAAA;QACnD,OAAO,cAAc,CAAC,IAAI,EAAE,KAAK,CAAC;YACjC,CAAC,CAAC,EAAE,GAAG,WAAW,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE;YACvC,CAAC,CAAC,IAAI,CAAA;IACR,CAAC;IACD,OAAO,EAAE,mBAAmB,EAAE,KAAK,EAAE,CAAA;AACtC,CAAC","sourcesContent":["import type { FetchedPage, FetchPageInput } from '../fetch-page.js'\nimport { asUnchanged, isUnchangedFor } from '../unchanged-page.js'\n\n/** A complete `FetchedPage`: an `ok` HTML read of `url` with nothing on it, plus `page`. */\nexport function fakePage(\n\tpage: Partial<FetchedPage> & { url: string }\n): FetchedPage {\n\treturn {\n\t\tfinalUrl: page.url,\n\t\tstatus: 200,\n\t\toutcome: 'ok',\n\t\tsource: 'workers-ai',\n\t\tcontentType: 'text/html; charset=utf-8',\n\t\tetag: null,\n\t\tlastModified: null,\n\t\tmarkdown: '',\n\t\ttitle: null,\n\t\tlang: null,\n\t\tcanonical: null,\n\t\talternates: [],\n\t\talternateLanguages: {},\n\t\tmetaDescription: null,\n\t\tjsonLd: [],\n\t\tlinks: [],\n\t\timages: [],\n\t\tdurationMs: 0,\n\t\tcaptureIssue: null,\n\t\thtmlHash: null,\n\t\t...page\n\t}\n}\n\n/**\n * A site in memory with the converter's `fetchPageAsMarkdown`: the pages\n * given by URL, `gone` (404) for any other, `unchanged` (304) when the input's\n * `etag`, `lastModified` or `htmlHash` matches the page's. `calls` lists every\n * input in order.\n */\nexport function fakeSite(\n\tpages: Record<string, Omit<Partial<FetchedPage>, 'url'>>\n) {\n\tconst calls: FetchPageInput[] = []\n\tasync function fetchPageAsMarkdown(input: FetchPageInput) {\n\t\tcalls.push(input)\n\t\tconst known = pages[input.url]\n\t\tif (!known) {\n\t\t\treturn fakePage({\n\t\t\t\turl: input.url,\n\t\t\t\tstatus: 404,\n\t\t\t\toutcome: 'gone',\n\t\t\t\tsource: null\n\t\t\t})\n\t\t}\n\t\tconst page = fakePage({ ...known, url: input.url })\n\t\treturn isUnchangedFor(page, input)\n\t\t\t? { ...asUnchanged(page), status: 304 }\n\t\t\t: page\n\t}\n\treturn { fetchPageAsMarkdown, calls }\n}\n"]}

package/dist/unchanged-page.d.ts

@@ -0,0 +1,9 @@
+import type { FetchedPage, FetchPageInput } from './fetch-page.js';
+/**
+ * Whether a full page read without validators is unchanged for a caller that
+ * holds these validators, decided the way the origin would have.
+ */
+export declare function isUnchangedFor(page: FetchedPage, input: FetchPageInput): boolean;
+/** `page` as the `unchanged` answer: validators and URLs kept, content dropped. */
+export declare function asUnchanged(page: FetchedPage): FetchedPage;
+//# sourceMappingURL=unchanged-page.d.ts.map

package/dist/unchanged-page.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"unchanged-page.d.ts","sourceRoot":"","sources":["../src/unchanged-page.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAElE;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,cAAc,WAStE;AAED,mFAAmF;AACnF,wBAAgB,WAAW,CAAC,IAAI,EAAE,WAAW,GAAG,WAAW,CAkB1D"}

package/dist/unchanged-page.js

@@ -0,0 +1,31 @@
+/**
+ * Whether a full page read without validators is unchanged for a caller that
+ * holds these validators, decided the way the origin would have.
+ */
+export function isUnchangedFor(page, input) {
+    return (page.outcome === 'ok' &&
+        Boolean((input.htmlHash && page.htmlHash === input.htmlHash) ||
+            (input.etag && page.etag === input.etag) ||
+            (input.lastModified && page.lastModified === input.lastModified)));
+}
+/** `page` as the `unchanged` answer: validators and URLs kept, content dropped. */
+export function asUnchanged(page) {
+    return {
+        ...page,
+        outcome: 'unchanged',
+        source: 'not-modified',
+        markdown: '',
+        title: null,
+        lang: null,
+        canonical: null,
+        alternates: [],
+        alternateLanguages: {},
+        metaDescription: null,
+        jsonLd: [],
+        links: [],
+        images: [],
+        captureIssue: null,
+        htmlHash: null
+    };
+}
+//# sourceMappingURL=unchanged-page.js.map

package/dist/unchanged-page.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"unchanged-page.js","sourceRoot":"","sources":["../src/unchanged-page.ts"],"names":[],"mappings":"AAEA;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,IAAiB,EAAE,KAAqB;IACtE,OAAO,CACN,IAAI,CAAC,OAAO,KAAK,IAAI;QACrB,OAAO,CACN,CAAC,KAAK,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,CAAC;YACpD,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC;YACxC,CAAC,KAAK,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,KAAK,KAAK,CAAC,YAAY,CAAC,CAChE,CACD,CAAA;AACF,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,WAAW,CAAC,IAAiB;IAC5C,OAAO;QACN,GAAG,IAAI;QACP,OAAO,EAAE,WAAW;QACpB,MAAM,EAAE,cAAc;QACtB,QAAQ,EAAE,EAAE;QACZ,KAAK,EAAE,IAAI;QACX,IAAI,EAAE,IAAI;QACV,SAAS,EAAE,IAAI;QACf,UAAU,EAAE,EAAE;QACd,kBAAkB,EAAE,EAAE;QACtB,eAAe,EAAE,IAAI;QACrB,MAAM,EAAE,EAAE;QACV,KAAK,EAAE,EAAE;QACT,MAAM,EAAE,EAAE;QACV,YAAY,EAAE,IAAI;QAClB,QAAQ,EAAE,IAAI;KACd,CAAA;AACF,CAAC","sourcesContent":["import type { FetchedPage, FetchPageInput } from './fetch-page.js'\n\n/**\n * Whether a full page read without validators is unchanged for a caller that\n * holds these validators, decided the way the origin would have.\n */\nexport function isUnchangedFor(page: FetchedPage, input: FetchPageInput) {\n\treturn (\n\t\tpage.outcome === 'ok' &&\n\t\tBoolean(\n\t\t\t(input.htmlHash && page.htmlHash === input.htmlHash) ||\n\t\t\t(input.etag && page.etag === input.etag) ||\n\t\t\t(input.lastModified && page.lastModified === input.lastModified)\n\t\t)\n\t)\n}\n\n/** `page` as the `unchanged` answer: validators and URLs kept, content dropped. */\nexport function asUnchanged(page: FetchedPage): FetchedPage {\n\treturn {\n\t\t...page,\n\t\toutcome: 'unchanged',\n\t\tsource: 'not-modified',\n\t\tmarkdown: '',\n\t\ttitle: null,\n\t\tlang: null,\n\t\tcanonical: null,\n\t\talternates: [],\n\t\talternateLanguages: {},\n\t\tmetaDescription: null,\n\t\tjsonLd: [],\n\t\tlinks: [],\n\t\timages: [],\n\t\tcaptureIssue: null,\n\t\thtmlHash: null\n\t}\n}\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "website-to-markdown",
-	"version": "1.1.1",
+	"version": "2.0.0",
 	"description": "Turn URLs, HTML and files into markdown, with sitemap discovery and per-host limits.",
 	"type": "module",
 	"license": "MIT",
@@ -35,6 +35,10 @@
 		"./schema": {
 			"types": "./dist/schema/index.d.ts",
 			"import": "./dist/schema/index.js"
+		},
+		"./testing": {
+			"types": "./dist/testing/index.d.ts",
+			"import": "./dist/testing/index.js"
 		}
 	},
 	"bin": {
@@ -54,9 +58,10 @@
 	"scripts": {
 		"build": "bun scripts/build.ts",
 		"test": "bun test",
-		"typecheck": "tsc --noEmit",
-		"lint": "bun run typecheck && biome check",
-		"lint:write": "bun run typecheck && biome check --write",
+		"typecheck": "bun check",
+		"fmt": "bun fmt --no-script '**/*.ts'",
+		"lint": "bun run typecheck && biome check && bun fmt --no-script --check '**/*.ts'",
+		"lint:write": "bun run typecheck && biome check --write && bun run fmt",
 		"attw": "attw --pack . --profile esm-only",
 		"publint": "publint",
 		"verify:package": "node scripts/verify-package.mjs"
@@ -66,7 +71,7 @@
 		"fflate": "^0.8.2"
 	},
 	"peerDependencies": {
-		"hanzio": "^3.3.0",
+		"hanzio": "^3.4.0",
 		"zod": "^4.0.0"
 	},
 	"devDependencies": {
@@ -74,7 +79,7 @@
 		"@biomejs/biome": "^2.2.4",
 		"@types/bun": "^1.2.0",
 		"@types/node": "^24",
-		"hanzio": "^3.3.0",
+		"hanzio": "^3.4.0",
 		"publint": "^0.3.12",
 		"typescript": "^5.9.2",
 		"zod": "^4.4.3"
@@ -82,5 +87,5 @@
 	"bugs": {
 		"url": "https://github.com/HansKristoffer/website-to-markdown/issues"
 	},
-	"packageManager": "bun@1.4.2"
+	"packageManager": "bun@1.4.4"
 }

package/src/cli.ts

@@ -63,7 +63,9 @@ export const cli = defineCli({
 				cloudflare: {
 					apiToken: CLOUDFLARE_API_TOKEN,
 					accountId: CLOUDFLARE_ACCOUNT_ID
-				}
+				},
+				// Run by a person on their own machine, so a dev server is fair game.
+				allowLocalhost: true
 			})
 
 			ctx.stderr.write(`Fetching ${url}…\n`)

package/src/content-type.ts

@@ -1,17 +1,19 @@
-const MARKDOWN_CONTENT_TYPES = new Set([
+import { isOneOf, type Nullable } from 'hanzio'
+
+const MARKDOWN_CONTENT_TYPES = [
 	'text/markdown',
 	'text/x-markdown',
 	'application/markdown'
-])
+] as const
 
-const HTML_CONTENT_TYPES = new Set(['text/html', 'application/xhtml+xml'])
+const HTML_CONTENT_TYPES = ['text/html', 'application/xhtml+xml'] as const
 
-const TEXTUAL_CONTENT_TYPES = new Set([
+const TEXTUAL_CONTENT_TYPES = [
 	'application/xml',
 	'application/json',
 	'application/rss+xml',
 	'application/atom+xml'
-])
+] as const
 
 /**
  * The formats Workers AI `toMarkdown` accepts as documents (its `/supported`
@@ -42,7 +44,7 @@ const CONVERTIBLE_DOCUMENT_EXTENSIONS = new Map([
 	['application/vnd.oasis.opendocument.presentation', 'odp']
 ])
 
-function normalizeContentType(contentType: string | null | undefined) {
+function normalizeContentType(contentType: Nullable<string>) {
 	return contentType?.split(';')[0]?.trim().toLowerCase() ?? ''
 }
 
@@ -50,7 +52,7 @@ function isTextualContentType(type: string) {
 	return (
 		type === '' ||
 		type.startsWith('text/') ||
-		TEXTUAL_CONTENT_TYPES.has(type) ||
+		isOneOf(TEXTUAL_CONTENT_TYPES, type) ||
 		// `application/ld+json`, `application/rdf+xml`, … but not `image/svg+xml`.
 		(type.startsWith('application/') &&
 			(type.endsWith('+json') || type.endsWith('+xml')))
@@ -69,8 +71,8 @@ export type ContentKind = 'markdown' | 'html' | 'text' | 'document' | 'asset'
 /** What the fetch layer should do with a body, decided from the header alone. */
 export function classifyContentType(contentType: string | null): ContentKind {
 	const type = normalizeContentType(contentType)
-	if (MARKDOWN_CONTENT_TYPES.has(type)) return 'markdown'
-	if (HTML_CONTENT_TYPES.has(type)) return 'html'
+	if (isOneOf(MARKDOWN_CONTENT_TYPES, type)) return 'markdown'
+	if (isOneOf(HTML_CONTENT_TYPES, type)) return 'html'
 	if (CONVERTIBLE_DOCUMENT_EXTENSIONS.has(type)) return 'document'
 	if (isTextualContentType(type)) return 'text'
 	return 'asset'