website-to-markdown 1.0.0 → 1.1.0

package/src/cli.ts

@@ -3,76 +3,98 @@
  * Fetch a public URL as markdown and print the result. Page metadata goes to
  * stderr; markdown goes to stdout.
  */
+import { readFile } from 'node:fs/promises'
+import { parseEnv } from 'node:util'
+import { type CliIO, CliError, defineCli, defineCommand } from 'hanzio/cli'
+import { z } from 'zod'
 import { createWebsiteToMarkdown } from './fetch-page.js'
 
-const HELP = `
-Fetch a public URL as markdown.
+export const cli = defineCli({
+	name: 'website-to-markdown',
+	commands: defineCommand({
+		description:
+			'Fetch a public URL as markdown. Needs CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID.',
+		input: z.object({
+			url: z
+				.string()
+				.describe('e.g. https://example.com/about')
+				.meta({ positional: true }),
+			render: z
+				.boolean()
+				.default(true)
+				.describe(
+					'Use the Browser Rendering tier; --no-render keeps to HTML tiers.'
+				),
+			envFile: z
+				.string()
+				.optional()
+				.describe('Read CLOUDFLARE_* from a dotenv file; the environment wins.')
+		}),
+		run: async ({ url, render, envFile }, ctx) => {
+			const file = envFile ? parseEnv(await readEnvFile(envFile)) : {}
+			const CLOUDFLARE_API_TOKEN =
+				ctx.env.CLOUDFLARE_API_TOKEN || file.CLOUDFLARE_API_TOKEN
+			const CLOUDFLARE_ACCOUNT_ID =
+				ctx.env.CLOUDFLARE_ACCOUNT_ID || file.CLOUDFLARE_ACCOUNT_ID
+			if (!CLOUDFLARE_API_TOKEN || !CLOUDFLARE_ACCOUNT_ID)
+				throw new CliError(
+					'Set CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID.'
+				)
 
-Usage:
-  website-to-markdown <url> [--no-render]
+			const { fetchPageAsMarkdown } = createWebsiteToMarkdown({
+				cloudflare: {
+					apiToken: CLOUDFLARE_API_TOKEN,
+					accountId: CLOUDFLARE_ACCOUNT_ID
+				}
+			})
 
-Options:
-  --no-render   Skip the Browser Rendering tier (HTML tiers only).
+			ctx.stderr.write(`Fetching ${url}…\n`)
 
-Environment:
-  CLOUDFLARE_API_TOKEN, CLOUDFLARE_ACCOUNT_ID
+			const page = await fetchPageAsMarkdown({ url, render })
 
-Example:
-  website-to-markdown https://example.com/about
-`.trim()
+			ctx.stderr.write(
+				`${JSON.stringify(
+					{
+						outcome: page.outcome,
+						status: page.status,
+						source: page.source,
+						url: page.url,
+						finalUrl: page.finalUrl,
+						contentType: page.contentType,
+						etag: page.etag,
+						lastModified: page.lastModified,
+						title: page.title,
+						lang: page.lang,
+						canonical: page.canonical,
+						metaDescription: page.metaDescription,
+						jsonLdBlocks: page.jsonLd.length,
+						links: page.links.length,
+						images: page.images.length,
+						markdownChars: page.markdown.length,
+						durationMs: page.durationMs
+					},
+					null,
+					2
+				)}\n`
+			)
 
-const args = Bun.argv.slice(2)
-const url = args.find((arg) => !arg.startsWith('-'))
-if (!url || args.includes('-h') || args.includes('--help')) {
-	console.log(HELP)
-	process.exit(url ? 0 : 1)
-}
-
-const { CLOUDFLARE_API_TOKEN, CLOUDFLARE_ACCOUNT_ID } = process.env
-if (!CLOUDFLARE_API_TOKEN || !CLOUDFLARE_ACCOUNT_ID) {
-	console.error('Set CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID.')
-	process.exit(1)
-}
-
-const { fetchPageAsMarkdown } = createWebsiteToMarkdown({
-	cloudflare: {
-		apiToken: CLOUDFLARE_API_TOKEN,
-		accountId: CLOUDFLARE_ACCOUNT_ID
-	}
+			if (page.outcome !== 'ok') ctx.exitCode = 1
+			return page.markdown
+		}
+	})
 })
 
-console.error(`Fetching ${url}…`)
-
-const page = await fetchPageAsMarkdown({
-	url,
-	render: !args.includes('--no-render')
-})
+function readEnvFile(path: string) {
+	return readFile(path, 'utf8').catch(() => {
+		throw new CliError(`Cannot read --env-file ${path}.`)
+	})
+}
 
-console.error(
-	JSON.stringify(
-		{
-			outcome: page.outcome,
-			status: page.status,
-			source: page.source,
-			url: page.url,
-			finalUrl: page.finalUrl,
-			contentType: page.contentType,
-			etag: page.etag,
-			lastModified: page.lastModified,
-			title: page.title,
-			lang: page.lang,
-			canonical: page.canonical,
-			metaDescription: page.metaDescription,
-			jsonLdBlocks: page.jsonLd.length,
-			links: page.links.length,
-			images: page.images.length,
-			markdownChars: page.markdown.length,
-			durationMs: page.durationMs
-		},
-		null,
-		2
-	)
-)
+/** The bin: 1.0 printed the help and exited 1 when no URL was given, even with `--help`. */
+export async function main(argv: readonly string[], io?: CliIO) {
+	if (argv.some((arg) => !arg.startsWith('-'))) return cli.run(argv, io)
+	await cli.run(['--help'], io)
+	return 1
+}
 
-console.log(page.markdown)
-if (page.outcome !== 'ok') process.exit(1)
+if (import.meta.main) process.exitCode = await main(process.argv.slice(2))

package/src/client.ts

@@ -1,9 +1,5 @@
 import Cloudflare from 'cloudflare'
-import {
-	type ApiError,
-	createApiClient,
-	HttpResponseError
-} from 'hanzio/api-wrapper'
+import { type ApiError, createApiClient } from 'hanzio/api-wrapper'
 import {
 	type ConcurrencyLimit,
 	type CreateConcurrencyLimit,
@@ -14,6 +10,7 @@ import {
 	CloudflareRateLimitedError,
 	withCloudflareRateLimitRetry
 } from './rate-limit-retry.js'
+import { isReportableApiError } from './report-api-error.js'
 import { rateLimitFromError } from './retry-after.js'
 import { toMarkdown } from './to-markdown.js'
 import { BROWSER_USER_AGENT } from './user-agent.js'
@@ -53,16 +50,17 @@ export function createCloudflareClient(options: {
 	createLimit: CreateConcurrencyLimit
 	/** Every Workers AI client error except a 429, which is retried instead. */
 	onApiError?: ((error: ApiError) => void) | undefined
+	/** Pass timeouts to `onApiError`; default true. */
+	reportTimeouts?: boolean | undefined
 	logger: Logger
 }) {
 	const { credentials, logger } = options
 	const limits = createCloudflareLimits(options.createLimit)
 
-	function reportUnlessRateLimited(error: ApiError) {
-		// Expected transient throttles: we retry and trip the shared cooldown.
-		// Reporting every 429 floods the error tracker without adding signal.
-		if (error instanceof HttpResponseError && error.status === 429) return
-		options.onApiError?.(error)
+	function report(error: ApiError) {
+		if (isReportableApiError(error, options.reportTimeouts)) {
+			options.onApiError?.(error)
+		}
 	}
 
 	let cloudflareSdk: Cloudflare | undefined
@@ -79,7 +77,7 @@ export function createCloudflareClient(options: {
 			},
 			timeoutMs: 30_000,
 			retries: 0,
-			onError: reportUnlessRateLimited,
+			onError: report,
 			endpoints: {
 				toMarkdown
 			}

package/src/crawl-snapshot.ts

@@ -3,6 +3,12 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises'
 import { resolve } from 'node:path'
 import { sha256Hex } from './hash.js'
 import type { Logger } from './logger.js'
+import {
+	CRAWL_SNAPSHOT_NAME_PATTERN,
+	type CrawlSnapshotInput
+} from './schema/index.js'
+
+export { CRAWL_SNAPSHOT_NAME_PATTERN, type CrawlSnapshotInput }
 
 /**
  * A crawl recorded once and replayed, for benchmarks. Two runs of a pipeline
@@ -19,32 +25,24 @@ import type { Logger } from './logger.js'
  * other tenants' jobs on the same lane. Never in production.
  */
 
-export type CrawlSnapshotInput = {
-	mode: 'record' | 'replay'
-	/** `<snapshot>/<host>`: the bench's snapshot label, then the site. */
-	name: string
-}
-
-/** The name becomes a path, so it may not climb out of `root`. */
-export const CRAWL_SNAPSHOT_NAME_PATTERN =
-	/^[a-z0-9._-]{1,80}\/[a-z0-9._-]{1,120}$/
-
 type CrawlSnapshot = CrawlSnapshotInput & {
 	directory: string
 	logger: Logger | undefined
 }
 
+export type CrawlSnapshotOptions = {
+	/** Directory the snapshots live under. */
+	root: string
+	/** Told about replay misses. */
+	logger?: Logger | undefined
+}
+
 const snapshot = new AsyncLocalStorage<CrawlSnapshot>()
 
 export function withCrawlSnapshot<T>(
 	input: CrawlSnapshotInput | null | undefined,
 	run: () => Promise<T>,
-	options: {
-		/** Directory the snapshots live under. */
-		root: string
-		/** Told about replay misses. */
-		logger?: Logger
-	}
+	options: CrawlSnapshotOptions
 ) {
 	if (!input) return run()
 	if (process.env.NODE_ENV === 'production') {

package/src/fetch-page.ts

@@ -10,7 +10,13 @@ import {
 	type ContentKind,
 	convertibleDocumentType
 } from './content-type.js'
-import { currentCrawlSnapshot, throughCrawlSnapshot } from './crawl-snapshot.js'
+import {
+	type CrawlSnapshotInput,
+	type CrawlSnapshotOptions,
+	currentCrawlSnapshot,
+	throughCrawlSnapshot,
+	withCrawlSnapshot
+} from './crawl-snapshot.js'
 import {
 	isApplicationShell,
 	isBlankMarkdown,
@@ -473,7 +479,18 @@ export type WebsiteToMarkdownOptions = {
 	createLimit?: CreateConcurrencyLimit
 	/** Workers AI client errors other than 429s, for an error tracker. */
 	onApiError?: (error: ApiError) => void
+	/**
+	 * Pass Workers AI timeouts to `onApiError`. A timed-out page falls through
+	 * to the next tier, so `false` keeps import bursts out of error tracking.
+	 * Default true.
+	 */
+	reportTimeouts?: boolean
 	logger?: Logger
+	/**
+	 * Where the returned `withCrawlSnapshot` keeps snapshots. Its logger
+	 * defaults to `logger`.
+	 */
+	crawlSnapshot?: CrawlSnapshotOptions
 	/** The OCR tier for a PDF Workers AI returned nothing for. */
 	parsePdf?: PdfParser
 	/**
@@ -494,6 +511,7 @@ export function createWebsiteToMarkdown(options: WebsiteToMarkdownOptions) {
 		credentials: options.cloudflare,
 		createLimit,
 		onApiError: options.onApiError,
+		reportTimeouts: options.reportTimeouts,
 		logger
 	})
 	const hostLimits = createHostLimits(createLimit)
@@ -1003,7 +1021,23 @@ export function createWebsiteToMarkdown(options: WebsiteToMarkdownOptions) {
 		convertDocumentBytes,
 		urlToMarkdown,
 		convertFileWithWorkersAi: client.convertFileWithWorkersAi,
-		renderUrlToMarkdown: client.renderUrlToMarkdown
+		renderUrlToMarkdown: client.renderUrlToMarkdown,
+		/** The SSRF guard the converter runs, bound to its `logger`. */
+		isHostAllowed: allowHost,
+		/** `withCrawlSnapshot` bound to the `crawlSnapshot` option. */
+		withCrawlSnapshot<T>(
+			input: CrawlSnapshotInput | null | undefined,
+			run: () => Promise<T>
+		) {
+			if (!input) return run()
+			if (!options.crawlSnapshot) {
+				throw new Error('Set crawlSnapshot.root to use crawl snapshots')
+			}
+			return withCrawlSnapshot(input, run, {
+				root: options.crawlSnapshot.root,
+				logger: options.crawlSnapshot.logger ?? logger
+			})
+		}
 	}
 }
 

package/src/index.ts

@@ -11,6 +11,7 @@ export {
 export {
 	CRAWL_SNAPSHOT_NAME_PATTERN,
 	type CrawlSnapshotInput,
+	type CrawlSnapshotOptions,
 	currentCrawlSnapshot,
 	throughCrawlSnapshot,
 	withCrawlSnapshot

package/src/limit.ts

@@ -1,8 +1,9 @@
 import { createMemoryConcurrencyLimit } from './memory-limit.js'
 
 /**
- * A semaphore: at most `max` callbacks in flight. hellu's Redis Lua limiter
- * (`createRedisConcurrencyLimit`) fits this shape as is; the default is
+ * A semaphore: at most `max` callbacks in flight. `hanzio/redis`'s
+ * `createRedisConcurrencyLimit` returns this shape (plus `queueDepth`, which is
+ * not needed here) once `send` is bound; the default is
  * {@link createMemoryConcurrencyLimit}.
  */
 export type ConcurrencyLimit = {

package/src/report-api-error.ts

@@ -0,0 +1,16 @@
+import {
+	type ApiError,
+	HttpResponseError,
+	RequestTimeoutError
+} from 'hanzio/api-wrapper'
+
+/**
+ * Whether `onApiError` hears about a Workers AI client error. A 429 never: it
+ * is retried and trips the shared cooldown, and reporting every one floods the
+ * error tracker. A timeout only with `reportTimeouts`, since the page falls
+ * through to the next tier anyway.
+ */
+export function isReportableApiError(error: ApiError, reportTimeouts = true) {
+	if (error instanceof HttpResponseError && error.status === 429) return false
+	return reportTimeouts || !(error instanceof RequestTimeoutError)
+}

package/src/schema/index.ts

@@ -0,0 +1,14 @@
+/**
+ * Values a shared schema can import without the runtime: no Node built-ins,
+ * no peer libraries.
+ */
+
+/** A crawl snapshot name, `<snapshot>/<host>`; it becomes a path, so it may not climb out of the root. */
+export const CRAWL_SNAPSHOT_NAME_PATTERN =
+	/^[a-z0-9._-]{1,80}\/[a-z0-9._-]{1,120}$/
+
+export type CrawlSnapshotInput = {
+	mode: 'record' | 'replay'
+	/** `<snapshot>/<host>`: the bench's snapshot label, then the site. */
+	name: string
+}