website-to-markdown 1.1.1 → 2.0.0

package/src/crawl-snapshot.ts

@@ -1,6 +1,7 @@
 import { AsyncLocalStorage } from 'node:async_hooks'
 import { mkdir, readFile, writeFile } from 'node:fs/promises'
 import { resolve } from 'node:path'
+import { invariant, type Nullable } from 'hanzio'
 import { sha256Hex } from './hash.js'
 import type { Logger } from './logger.js'
 import {
@@ -40,17 +41,19 @@ export type CrawlSnapshotOptions = {
 const snapshot = new AsyncLocalStorage<CrawlSnapshot>()
 
 export function withCrawlSnapshot<T>(
-	input: CrawlSnapshotInput | null | undefined,
+	input: Nullable<CrawlSnapshotInput>,
 	run: () => Promise<T>,
 	options: CrawlSnapshotOptions
 ) {
 	if (!input) return run()
-	if (process.env.NODE_ENV === 'production') {
-		throw new Error('A crawl snapshot is a local bench tool')
-	}
-	if (!CRAWL_SNAPSHOT_NAME_PATTERN.test(input.name)) {
-		throw new Error(`Invalid crawl snapshot name: ${input.name}`)
-	}
+	invariant(
+		process.env.NODE_ENV !== 'production',
+		'A crawl snapshot is a local bench tool'
+	)
+	invariant(
+		CRAWL_SNAPSHOT_NAME_PATTERN.test(input.name),
+		`Invalid crawl snapshot name: ${input.name}`
+	)
 
 	return snapshot.run(
 		{
@@ -79,7 +82,9 @@ export async function throughCrawlSnapshot<T>(
 	const file = resolve(active.directory, `${sha256Hex(key).slice(0, 32)}.json`)
 	if (active.mode === 'replay') {
 		const stored = await readFile(file, 'utf8').catch(() => null)
-		if (stored !== null) return (JSON.parse(stored) as { value: T }).value
+		// A file cut short by a killed recording is a miss, not a crash.
+		const parsed = stored === null ? null : parseStored<T>(stored)
+		if (parsed) return parsed.value
 		active.logger?.warn('crawl-snapshot.miss', { key })
 	}
 
@@ -89,6 +94,14 @@ export async function throughCrawlSnapshot<T>(
 	return value
 }
 
+function parseStored<T>(stored: string) {
+	try {
+		return JSON.parse(stored) as { value: T }
+	} catch {
+		return null
+	}
+}
+
 /** The snapshot this job reads or writes, to hand on to the job it queues. */
 export function currentCrawlSnapshot(): CrawlSnapshotInput | null {
 	const active = snapshot.getStore()