website-to-markdown 1.0.1 → 1.1.0

package/src/crawl-snapshot.ts

@@ -3,6 +3,12 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises'
 import { resolve } from 'node:path'
 import { sha256Hex } from './hash.js'
 import type { Logger } from './logger.js'
+import {
+	CRAWL_SNAPSHOT_NAME_PATTERN,
+	type CrawlSnapshotInput
+} from './schema/index.js'
+
+export { CRAWL_SNAPSHOT_NAME_PATTERN, type CrawlSnapshotInput }
 
 /**
  * A crawl recorded once and replayed, for benchmarks. Two runs of a pipeline
@@ -19,32 +25,24 @@ import type { Logger } from './logger.js'
  * other tenants' jobs on the same lane. Never in production.
  */
 
-export type CrawlSnapshotInput = {
-	mode: 'record' | 'replay'
-	/** `<snapshot>/<host>`: the bench's snapshot label, then the site. */
-	name: string
-}
-
-/** The name becomes a path, so it may not climb out of `root`. */
-export const CRAWL_SNAPSHOT_NAME_PATTERN =
-	/^[a-z0-9._-]{1,80}\/[a-z0-9._-]{1,120}$/
-
 type CrawlSnapshot = CrawlSnapshotInput & {
 	directory: string
 	logger: Logger | undefined
 }
 
+export type CrawlSnapshotOptions = {
+	/** Directory the snapshots live under. */
+	root: string
+	/** Told about replay misses. */
+	logger?: Logger | undefined
+}
+
 const snapshot = new AsyncLocalStorage<CrawlSnapshot>()
 
 export function withCrawlSnapshot<T>(
 	input: CrawlSnapshotInput | null | undefined,
 	run: () => Promise<T>,
-	options: {
-		/** Directory the snapshots live under. */
-		root: string
-		/** Told about replay misses. */
-		logger?: Logger
-	}
+	options: CrawlSnapshotOptions
 ) {
 	if (!input) return run()
 	if (process.env.NODE_ENV === 'production') {

package/src/fetch-page.ts

@@ -10,7 +10,13 @@ import {
 	type ContentKind,
 	convertibleDocumentType
 } from './content-type.js'
-import { currentCrawlSnapshot, throughCrawlSnapshot } from './crawl-snapshot.js'
+import {
+	type CrawlSnapshotInput,
+	type CrawlSnapshotOptions,
+	currentCrawlSnapshot,
+	throughCrawlSnapshot,
+	withCrawlSnapshot
+} from './crawl-snapshot.js'
 import {
 	isApplicationShell,
 	isBlankMarkdown,
@@ -473,7 +479,18 @@ export type WebsiteToMarkdownOptions = {
 	createLimit?: CreateConcurrencyLimit
 	/** Workers AI client errors other than 429s, for an error tracker. */
 	onApiError?: (error: ApiError) => void
+	/**
+	 * Pass Workers AI timeouts to `onApiError`. A timed-out page falls through
+	 * to the next tier, so `false` keeps import bursts out of error tracking.
+	 * Default true.
+	 */
+	reportTimeouts?: boolean
 	logger?: Logger
+	/**
+	 * Where the returned `withCrawlSnapshot` keeps snapshots. Its logger
+	 * defaults to `logger`.
+	 */
+	crawlSnapshot?: CrawlSnapshotOptions
 	/** The OCR tier for a PDF Workers AI returned nothing for. */
 	parsePdf?: PdfParser
 	/**
@@ -494,6 +511,7 @@ export function createWebsiteToMarkdown(options: WebsiteToMarkdownOptions) {
 		credentials: options.cloudflare,
 		createLimit,
 		onApiError: options.onApiError,
+		reportTimeouts: options.reportTimeouts,
 		logger
 	})
 	const hostLimits = createHostLimits(createLimit)
@@ -1003,7 +1021,23 @@ export function createWebsiteToMarkdown(options: WebsiteToMarkdownOptions) {
 		convertDocumentBytes,
 		urlToMarkdown,
 		convertFileWithWorkersAi: client.convertFileWithWorkersAi,
-		renderUrlToMarkdown: client.renderUrlToMarkdown
+		renderUrlToMarkdown: client.renderUrlToMarkdown,
+		/** The SSRF guard the converter runs, bound to its `logger`. */
+		isHostAllowed: allowHost,
+		/** `withCrawlSnapshot` bound to the `crawlSnapshot` option. */
+		withCrawlSnapshot<T>(
+			input: CrawlSnapshotInput | null | undefined,
+			run: () => Promise<T>
+		) {
+			if (!input) return run()
+			if (!options.crawlSnapshot) {
+				throw new Error('Set crawlSnapshot.root to use crawl snapshots')
+			}
+			return withCrawlSnapshot(input, run, {
+				root: options.crawlSnapshot.root,
+				logger: options.crawlSnapshot.logger ?? logger
+			})
+		}
 	}
 }
 

package/src/index.ts

@@ -11,6 +11,7 @@ export {
 export {
 	CRAWL_SNAPSHOT_NAME_PATTERN,
 	type CrawlSnapshotInput,
+	type CrawlSnapshotOptions,
 	currentCrawlSnapshot,
 	throughCrawlSnapshot,
 	withCrawlSnapshot

package/src/limit.ts

@@ -1,8 +1,9 @@
 import { createMemoryConcurrencyLimit } from './memory-limit.js'
 
 /**
- * A semaphore: at most `max` callbacks in flight. hellu's Redis Lua limiter
- * (`createRedisConcurrencyLimit`) fits this shape as is; the default is
+ * A semaphore: at most `max` callbacks in flight. `hanzio/redis`'s
+ * `createRedisConcurrencyLimit` returns this shape (plus `queueDepth`, which is
+ * not needed here) once `send` is bound; the default is
  * {@link createMemoryConcurrencyLimit}.
  */
 export type ConcurrencyLimit = {

package/src/report-api-error.ts

@@ -0,0 +1,16 @@
+import {
+	type ApiError,
+	HttpResponseError,
+	RequestTimeoutError
+} from 'hanzio/api-wrapper'
+
+/**
+ * Whether `onApiError` hears about a Workers AI client error. A 429 never: it
+ * is retried and trips the shared cooldown, and reporting every one floods the
+ * error tracker. A timeout only with `reportTimeouts`, since the page falls
+ * through to the next tier anyway.
+ */
+export function isReportableApiError(error: ApiError, reportTimeouts = true) {
+	if (error instanceof HttpResponseError && error.status === 429) return false
+	return reportTimeouts || !(error instanceof RequestTimeoutError)
+}

package/src/schema/index.ts

@@ -0,0 +1,14 @@
+/**
+ * Values a shared schema can import without the runtime: no Node built-ins,
+ * no peer libraries.
+ */
+
+/** A crawl snapshot name, `<snapshot>/<host>`; it becomes a path, so it may not climb out of the root. */
+export const CRAWL_SNAPSHOT_NAME_PATTERN =
+	/^[a-z0-9._-]{1,80}\/[a-z0-9._-]{1,120}$/
+
+export type CrawlSnapshotInput = {
+	mode: 'record' | 'replay'
+	/** `<snapshot>/<host>`: the bench's snapshot label, then the site. */
+	name: string
+}