website-to-markdown 1.1.0 → 2.0.0

package/src/retry-after.ts

@@ -1,13 +1,11 @@
-import { parseRetryAfter } from 'hanzio'
+import { clamp, isFunction, isRecord, parseRetryAfter } from 'hanzio'
 
 const DEFAULT_RETRY_AFTER_MS = 1_000
 const MAX_RETRY_AFTER_MS = 5 * 60 * 1_000
 
 function headerGet(headers: unknown, name: string): string | null {
-	if (!headers || typeof headers !== 'object') return null
-	if (typeof (headers as { get?: unknown }).get === 'function') {
-		return (headers as { get: (key: string) => string | null }).get(name)
-	}
+	if (!isRecord(headers)) return null
+	if (isFunction(headers.get)) return headers.get(name)
 	const record = headers as Record<string, string>
 	return record[name] ?? record[name.toLowerCase()] ?? null
 }
@@ -20,23 +18,17 @@ function headerGet(headers: unknown, name: string): string | null {
 export function rateLimitFromError(
 	error: unknown
 ): { retryAfterMs: number | null } | null {
-	if (typeof error !== 'object' || !error) return null
-	const status =
-		'status' in error && typeof error.status === 'number' ? error.status : null
-	if (status !== 429) return null
+	if (!isRecord(error) || error.status !== 429) return null
 
 	const raw =
-		headerGet('headers' in error ? error.headers : null, 'retry-after') ??
-		headerGet(
-			'responseHeaders' in error ? error.responseHeaders : null,
-			'retry-after'
-		)
+		headerGet(error.headers, 'retry-after') ??
+		headerGet(error.responseHeaders, 'retry-after')
 	const parsed = parseRetryAfter(raw)
 	return {
 		retryAfterMs:
 			parsed === undefined
 				? null
-				: Math.min(MAX_RETRY_AFTER_MS, Math.ceil(parsed))
+				: clamp(Math.ceil(parsed), 0, MAX_RETRY_AFTER_MS)
 	}
 }
 
@@ -44,8 +36,9 @@ export function rateLimitFromError(
 export function retryAfterMsFromError(error: unknown) {
 	const info = rateLimitFromError(error)
 	if (!info) return null
-	return Math.min(
-		MAX_RETRY_AFTER_MS,
-		Math.ceil(info.retryAfterMs ?? DEFAULT_RETRY_AFTER_MS)
+	return clamp(
+		Math.ceil(info.retryAfterMs ?? DEFAULT_RETRY_AFTER_MS),
+		0,
+		MAX_RETRY_AFTER_MS
 	)
 }

package/src/schema/index.ts

@@ -12,3 +12,37 @@ export type CrawlSnapshotInput = {
 	/** `<snapshot>/<host>`: the bench's snapshot label, then the site. */
 	name: string
 }
+
+/** Every `FetchedPage.outcome`, for narrowing a value read back from storage. */
+export const FETCH_OUTCOMES = [
+	'ok',
+	'unchanged',
+	'gone',
+	'blocked',
+	'transient',
+	'unsupported'
+] as const
+export type FetchOutcome = (typeof FETCH_OUTCOMES)[number]
+
+/** Every `FetchedPage.source`. `firecrawl-parse` is the `parsePdf` tier, whatever parser it is. */
+export const FETCH_SOURCES = [
+	'content-negotiation',
+	'workers-ai',
+	'browser-rendering',
+	'not-modified',
+	'firecrawl-parse'
+] as const
+export type FetchSource = (typeof FETCH_SOURCES)[number]
+
+/** Every `FetchedPage.captureIssue`: why a page came back incomplete or not at all. */
+export const CAPTURE_ISSUES = [
+	'markdown_limit',
+	'html_byte_limit',
+	'document_byte_limit',
+	'workers_ai_rate_limited',
+	'browser_rendering_rate_limited',
+	'render_incomplete',
+	'pdf_page_limit_unverified',
+	'pdf_extraction_failed'
+] as const
+export type CaptureIssue = (typeof CAPTURE_ISSUES)[number]

package/src/sitemap/index.ts

@@ -1,9 +1,31 @@
-import { escapeRegExp } from 'hanzio'
+import { getUniqueValues, isOneOf, trimTrailingSlash } from 'hanzio'
+import { isAllowedByRobots, parseRobotsTxt } from 'hanzio/sitemap'
 import { throughCrawlSnapshot } from '../crawl-snapshot.js'
+import { type Fetch, guardedFetch } from '../net-guard.js'
+import { decodeText, readBody } from '../read-body.js'
+import { BROWSER_USER_AGENT } from '../user-agent.js'
+
+export type { Fetch }
 
 const FETCH_TIMEOUT_MS = 10_000
 const MAX_SITEMAP_FILES = 20
 const MAX_URLS = 20_000
+/** Google reads the first 500 KiB of a robots.txt and ignores the rest. */
+const MAX_ROBOTS_BYTES = 500 * 1024
+/**
+ * Read cap per sitemap file. 20,000 URLs fit in a few MB; a longer file is
+ * read up to here and its complete entries used.
+ */
+const MAX_SITEMAP_BYTES = 10 * 1024 * 1024
+
+export type SitemapOptions = {
+	/**
+	 * The `fetch` robots.txt and sitemaps go through. Default
+	 * {@link guardedFetch} with the default guard (public addresses only, every
+	 * redirect checked); pass `converter.guardedFetch` to use the converter's.
+	 */
+	fetch?: Fetch | undefined
+}
 
 /** Conventional sitemap locations, tried in order when robots.txt names none. */
 const SITEMAP_ROOT_PATHS = [
@@ -64,28 +86,41 @@ export function parseSitemapXml(xml: string) {
 	return { pages, nested }
 }
 
-function fetchText(url: string, accept: string) {
-	return throughCrawlSnapshot(`text ${accept} ${url}`, () =>
-		fetchTextLive(url, accept)
+type TextRequest = { accept: string; maxBytes: number; fetch: Fetch }
+
+function fetchText(url: string, request: TextRequest) {
+	return throughCrawlSnapshot(`text ${request.accept} ${url}`, () =>
+		fetchTextLive(url, request)
 	)
 }
 
-async function fetchTextLive(url: string, accept: string) {
+async function fetchTextLive(url: string, request: TextRequest) {
 	try {
-		const response = await fetch(url, {
+		const response = await request.fetch(url, {
 			signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
-			headers: { accept }
+			headers: { accept: request.accept, 'user-agent': BROWSER_USER_AGENT }
 		})
-		if (!response.ok) return null
 		const type = response.headers.get('content-type') ?? ''
 		// A CMS that renders its 404 page for a missing file answers 200 with HTML.
-		if (type.includes('text/html')) return null
-		return await response.text()
+		if (!response.ok || type.includes('text/html')) {
+			void response.body?.cancel().catch(() => {})
+			return null
+		}
+		return decodeText((await readBody(response, request.maxBytes)).bytes)
 	} catch {
 		return null
 	}
 }
 
+const ROBOTS_REQUEST = {
+	accept: 'text/plain,*/*;q=0.8',
+	maxBytes: MAX_ROBOTS_BYTES
+}
+const SITEMAP_REQUEST = {
+	accept: 'application/xml,text/xml;q=0.9,*/*;q=0.8',
+	maxBytes: MAX_SITEMAP_BYTES
+}
+
 function originOf(websiteUrl: string) {
 	try {
 		return new URL(websiteUrl).origin
@@ -100,12 +135,18 @@ function originOf(websiteUrl: string) {
  * here), and its rules. An HTML body or any non-200 means "no robots.txt";
  * never throws.
  */
-export async function fetchRobots(websiteUrl: string) {
+export async function fetchRobots(
+	websiteUrl: string,
+	options: SitemapOptions = {}
+) {
 	const sitemaps: string[] = []
 	const origin = originOf(websiteUrl)
 	if (!origin) return { sitemaps, rules: [] }
 	const robotsUrl = `${origin}/robots.txt`
-	const body = await fetchText(robotsUrl, 'text/plain,*/*;q=0.8')
+	const body = await fetchText(robotsUrl, {
+		...ROBOTS_REQUEST,
+		fetch: options.fetch ?? guardedFetch
+	})
 	if (!body) return { sitemaps, rules: [] }
 
 	for (const match of body.matchAll(ROBOTS_SITEMAP_RE)) {
@@ -114,15 +155,16 @@ export async function fetchRobots(websiteUrl: string) {
 		try {
 			const absolute = new URL(value, robotsUrl)
 			// Only what we could actually GET - `ftp:` and friends are noise.
-			if (absolute.protocol !== 'http:' && absolute.protocol !== 'https:')
-				continue
-			const href = absolute.toString()
-			if (!sitemaps.includes(href)) sitemaps.push(href)
+			if (!isOneOf(['http:', 'https:'], absolute.protocol)) continue
+			sitemaps.push(absolute.toString())
 		} catch {
 			// A malformed Sitemap: line must not cost us the other lines.
 		}
 	}
-	return { sitemaps, rules: parseRobotsRules(body) }
+	return {
+		sitemaps: getUniqueValues(sitemaps),
+		rules: parseRobotsRules(body)
+	}
 }
 
 export type RobotsRule = { allow: boolean; pattern: string }
@@ -133,75 +175,27 @@ export type RobotsRule = { allow: boolean; pattern: string }
  * the wildcard group is what the site asks of everyone. An empty `Disallow:`
  * allows everything and adds no rule.
  */
-export function parseRobotsRules(body: string) {
-	const rules: RobotsRule[] = []
-	let agents: string[] = []
-	let inRules = false
-
-	for (const raw of body.split(/\r?\n/)) {
-		const match = /^\s*([a-z-]+)\s*:\s*(.*?)\s*$/i.exec(raw.replace(/#.*$/, ''))
-		if (!match) continue
-		const field = (match[1] ?? '').toLowerCase()
-		const value = match[2] ?? ''
-
-		if (field === 'user-agent') {
-			// A user-agent line after rules opens a new group.
-			if (inRules) agents = []
-			inRules = false
-			agents.push(value.toLowerCase())
-			continue
-		}
-		if (field !== 'allow' && field !== 'disallow') continue
-
-		inRules = true
-		if (agents.includes('*') && value !== '')
-			rules.push({ allow: field === 'allow', pattern: value })
-	}
-	return rules
-}
-
-/** Compiled once per pattern: the crawl asks for every link of every page. */
-const robotsPatternCache = new Map<string, RegExp>()
-
-function robotsPatternRegex(pattern: string) {
-	const cached = robotsPatternCache.get(pattern)
-	if (cached) return cached
-
-	const anchored = pattern.endsWith('$')
-	const body = (anchored ? pattern.slice(0, -1) : pattern)
-		.split('*')
-		.map((part) => escapeRegExp(part))
-		.join('.*')
-	const regex = new RegExp(`^${body}${anchored ? '$' : ''}`)
-	// Every site's patterns land here; a long-lived worker starts over rather than grow.
-	if (robotsPatternCache.size >= 5_000) robotsPatternCache.clear()
-	robotsPatternCache.set(pattern, regex)
-	return regex
+export function parseRobotsRules(body: string): RobotsRule[] {
+	const { allow, disallow } = parseRobotsTxt(body)
+	return [
+		...allow.map((pattern) => ({ allow: true, pattern })),
+		...disallow.map((pattern) => ({ allow: false, pattern }))
+	]
 }
 
 /**
  * Whether robots.txt lets us read this URL: the longest matching rule wins and
  * `Allow` wins a tie, as Google reads it. Shopify disallows `/cart/`, and the
- * crawler had fetched `/cart/add?id=…&quantity=1` - a GET that fills a cart
- * (Good Habit audit §3.5).
+ * crawler had fetched `/cart/add?id=…&quantity=1` - a GET that fills a cart.
  */
 export function robotsAllows(rules: RobotsRule[], url: string) {
-	let path: string
-	try {
-		const parsed = new URL(url)
-		path = parsed.pathname + parsed.search
-	} catch {
-		return true
-	}
-
-	let best: RobotsRule | null = null
-	for (const rule of rules) {
-		if (!robotsPatternRegex(rule.pattern).test(path)) continue
-		if (!best || rule.pattern.length > best.pattern.length) best = rule
-		else if (rule.pattern.length === best.pattern.length && rule.allow)
-			best = rule
-	}
-	return best?.allow ?? true
+	return isAllowedByRobots(
+		{
+			allow: rules.filter((rule) => rule.allow).map((rule) => rule.pattern),
+			disallow: rules.filter((rule) => !rule.allow).map((rule) => rule.pattern)
+		},
+		url
+	)
 }
 
 /**
@@ -212,7 +206,8 @@ export function robotsAllows(rules: RobotsRule[], url: string) {
 async function readSitemapTree(
 	roots: string[],
 	found: Map<string, { url: string; lastmod: string | null }>,
-	seen: Set<string>
+	seen: Set<string>,
+	fetch: Fetch
 ) {
 	let readable = false
 	let level = [...roots]
@@ -230,7 +225,7 @@ async function readSitemapTree(
 		}
 		const bodies = await Promise.all(
 			batch.map((sitemapUrl) =>
-				fetchText(sitemapUrl, 'application/xml,text/xml;q=0.9,*/*;q=0.8')
+				fetchText(sitemapUrl, { ...SITEMAP_REQUEST, fetch })
 			)
 		)
 		const next: string[] = []
@@ -263,10 +258,12 @@ async function readSitemapTree(
  */
 export async function discoverSitemapUrls(
 	websiteUrl: string,
-	robotsSitemaps: string[]
+	robotsSitemaps: string[],
+	options: SitemapOptions = {}
 ) {
 	const origin = originOf(websiteUrl)
 	if (!origin) return []
+	const fetch = options.fetch ?? guardedFetch
 	const found = new Map<string, { url: string; lastmod: string | null }>()
 	// Shared across roots so a broken sitemap index cannot spend the file budget
 	// several times over.
@@ -276,12 +273,12 @@ export async function discoverSitemapUrls(
 	// so all of them are read together - only the conventional roots compete.
 	if (
 		robotsSitemaps.length > 0 &&
-		(await readSitemapTree(robotsSitemaps, found, seen))
+		(await readSitemapTree(robotsSitemaps, found, seen, fetch))
 	) {
 		return [...found.values()]
 	}
 	for (const path of SITEMAP_ROOT_PATHS) {
-		if (await readSitemapTree([`${origin}${path}`], found, seen)) break
+		if (await readSitemapTree([`${origin}${path}`], found, seen, fetch)) break
 	}
 	return [...found.values()]
 }
@@ -291,8 +288,8 @@ export function normalizeSitemapUrl(url: string) {
 	try {
 		const parsed = new URL(url)
 		parsed.hash = ''
-		return parsed.toString().replace(/\/+$/, '')
+		return trimTrailingSlash(parsed.toString())
 	} catch {
-		return url.replace(/\/+$/, '')
+		return trimTrailingSlash(url)
 	}
 }

package/src/testing/index.ts

@@ -0,0 +1,60 @@
+import type { FetchedPage, FetchPageInput } from '../fetch-page.js'
+import { asUnchanged, isUnchangedFor } from '../unchanged-page.js'
+
+/** A complete `FetchedPage`: an `ok` HTML read of `url` with nothing on it, plus `page`. */
+export function fakePage(
+	page: Partial<FetchedPage> & { url: string }
+): FetchedPage {
+	return {
+		finalUrl: page.url,
+		status: 200,
+		outcome: 'ok',
+		source: 'workers-ai',
+		contentType: 'text/html; charset=utf-8',
+		etag: null,
+		lastModified: null,
+		markdown: '',
+		title: null,
+		lang: null,
+		canonical: null,
+		alternates: [],
+		alternateLanguages: {},
+		metaDescription: null,
+		jsonLd: [],
+		links: [],
+		images: [],
+		durationMs: 0,
+		captureIssue: null,
+		htmlHash: null,
+		...page
+	}
+}
+
+/**
+ * A site in memory with the converter's `fetchPageAsMarkdown`: the pages
+ * given by URL, `gone` (404) for any other, `unchanged` (304) when the input's
+ * `etag`, `lastModified` or `htmlHash` matches the page's. `calls` lists every
+ * input in order.
+ */
+export function fakeSite(
+	pages: Record<string, Omit<Partial<FetchedPage>, 'url'>>
+) {
+	const calls: FetchPageInput[] = []
+	async function fetchPageAsMarkdown(input: FetchPageInput) {
+		calls.push(input)
+		const known = pages[input.url]
+		if (!known) {
+			return fakePage({
+				url: input.url,
+				status: 404,
+				outcome: 'gone',
+				source: null
+			})
+		}
+		const page = fakePage({ ...known, url: input.url })
+		return isUnchangedFor(page, input)
+			? { ...asUnchanged(page), status: 304 }
+			: page
+	}
+	return { fetchPageAsMarkdown, calls }
+}

package/src/unchanged-page.ts

@@ -0,0 +1,37 @@
+import type { FetchedPage, FetchPageInput } from './fetch-page.js'
+
+/**
+ * Whether a full page read without validators is unchanged for a caller that
+ * holds these validators, decided the way the origin would have.
+ */
+export function isUnchangedFor(page: FetchedPage, input: FetchPageInput) {
+	return (
+		page.outcome === 'ok' &&
+		Boolean(
+			(input.htmlHash && page.htmlHash === input.htmlHash) ||
+			(input.etag && page.etag === input.etag) ||
+			(input.lastModified && page.lastModified === input.lastModified)
+		)
+	)
+}
+
+/** `page` as the `unchanged` answer: validators and URLs kept, content dropped. */
+export function asUnchanged(page: FetchedPage): FetchedPage {
+	return {
+		...page,
+		outcome: 'unchanged',
+		source: 'not-modified',
+		markdown: '',
+		title: null,
+		lang: null,
+		canonical: null,
+		alternates: [],
+		alternateLanguages: {},
+		metaDescription: null,
+		jsonLd: [],
+		links: [],
+		images: [],
+		captureIssue: null,
+		htmlHash: null
+	}
+}