pipework 0.251.0 → 0.252.1

package/src/infra/crawl/handle.ts

@@ -0,0 +1,693 @@
+import { Readable } from 'node:stream'
+import { sql } from '../../data/db/sql.js'
+import { pipeSystem } from '../../data/db/db.js'
+import type { DB } from '../../data/db/index.js'
+import { fitting } from '../../request/di/index.js'
+import type { Handler } from '../../request/di/types.js'
+import { getContext } from '../../request/context/store.js'
+import { resolveActorId } from '../../data/write/audit/actor.js'
+import { schema } from '../../data/schema/namespace.js'
+import { __flowStep } from '../../trace/runtime/flow-step.js'
+import { getBaseLogger } from '../../core/logging/index.js'
+import { ConflictError, NotFoundError } from '../../core/errors/index.js'
+import type { Manifold } from '../../core/pipework.js'
+import type { Queue } from '../../async/jobs/queue.js'
+import { untrustedGet } from '../fetch/handle.js'
+import type { FetchTarget } from '../fetch/handle.js'
+import type { Storage } from '../storage/handle.js'
+import type { Socket } from 'node:net'
+import { auditCrawl } from './audit.js'
+import { CrawlNotEnabledError, CrawlSeedError } from './errors.js'
+import { extractLinks } from './links.js'
+import { ALLOW_ALL, DISALLOW_ALL, parseRobots, type RobotsRules } from './robots.js'
+
+export type CrawlScope = 'host' | { readonly domain: string }
+export type CrawlState = 'awaiting_approval' | 'running' | 'done' | 'stopped' | 'failed'
+export type CrawlStoppedBy = 'pages' | 'depth' | 'bytes' | 'wall' | 'robots_unreachable'
+
+/** Limits of one crawl; each is optional and the default is in the field's description. */
+export interface CrawlLimits {
+  /** Requests in flight at once to one host. Default 1. */
+  readonly perHostConcurrency?: number
+  /** Least milliseconds between requests to a host; the gap used is the larger of this and the host's `Crawl-delay`. Default 1000. */
+  readonly minGapMs?: number
+  /** Most pages and files requested (robots.txt not counted). Default 1000. */
+  readonly maxPages?: number
+  /** Most links deep from the seed. Default 10. */
+  readonly maxDepth?: number
+  /** Most body bytes fetched in all. Default 1 GiB. */
+  readonly maxBytes?: number
+  /** Most bytes of one response; a larger one is refused as `too_large`. Default 25 MiB. */
+  readonly maxBytesPerResponse?: number
+  /** Most milliseconds the crawl runs, counted from its start. Default one hour. */
+  readonly maxWallMs?: number
+  /** Time limit of one request. Default 30 s. */
+  readonly requestTimeoutMs?: number
+  /** Most redirects followed in a chain. Default 5. */
+  readonly maxRedirects?: number
+  /** Time limit of one `onPage` call. Default 5000. */
+  readonly pageHookMs?: number
+}
+
+export type ResolvedCrawlLimits = Required<CrawlLimits>
+
+export interface CrawlPage {
+  readonly url: string
+  /** The page's html, already limited to `maxBytesPerResponse`. */
+  readonly html: string
+  /** Aborted when the hook has overrun `pageHookMs`. */
+  readonly signal: AbortSignal
+}
+
+export interface CrawlConfig {
+  /** The app's manifold; its Blueprint must have a `crawl` block. */
+  readonly instance: Manifold
+  /** Names the site and its job type (`pipework.crawl.<name>`): letters, digits, `_`, `.` and `-`, starting with a letter. */
+  readonly name: string
+  /** The queue the crawl job runs on; its worker surface must run `site.job`. */
+  readonly queue: Pick<Queue, 'enqueue'>
+  /** Which resources are listed. In-scope `text/html` is followed for links whatever this says; any other type this refuses is not read. */
+  readonly accept: (resource: { url: string; contentType: string }) => boolean
+  /** Called with each followed page's html. A hook that throws or overruns `pageHookMs` is recorded against the URL and the crawl goes on. */
+  readonly onPage?: (page: CrawlPage) => Promise<void> | void
+  /** When given, each listed resource's bytes are put here (the job's tenant) and the row records the sha256 and file id. */
+  readonly store?: Storage
+  /** Sent on every request; its product token (the leading letters) picks the robots.txt group. */
+  readonly userAgent: string
+  /** For tests: replaces DNS and the socket, as in `untrustedFetch.get`. */
+  readonly network?: {
+    readonly resolve?: (hostname: string) => Promise<string[]>
+    readonly connect?: (target: FetchTarget) => Socket
+  }
+  /** Record the full URL in each request's trace step. Default true: crawled links are public. */
+  readonly traceUrls?: boolean
+  /** Database holding the crawl tables; default the default database. */
+  readonly database?: string
+  /** Attempts the queue gives the job: a job that dies or throws is retried and carries on from the queued URLs. Default 5. */
+  readonly maxAttempts?: number
+  /** First wait before a robots.txt retry, doubling each time. Default 2000. */
+  readonly robotsRetryMs?: number
+}
+
+export interface CrawlStartOptions {
+  readonly seed: string
+  /** `'host'` (default): the seed's exact host. `{ domain: 'example.gov.uk' }`: that host and every host under it. */
+  readonly scope?: CrawlScope
+  readonly limits?: CrawlLimits
+}
+
+export interface CrawlCounts {
+  readonly queued: number
+  readonly fetching: number
+  readonly done: number
+  readonly refused: number
+  readonly listed: number
+  readonly unchanged: number
+  /** Body bytes fetched. */
+  readonly bytes: number
+}
+
+export interface CrawlStatus {
+  readonly state: CrawlState
+  /** What the crawl stays inside. A `'host'` scope moves to the new host when the seed's own redirect chain leaves the seed's host (example.gov.uk to www.example.gov.uk). */
+  readonly scope: { readonly host: string } | { readonly domain: string }
+  /** Present when `state` is `stopped`. */
+  readonly stoppedBy?: CrawlStoppedBy
+  /** Present when `state` is `failed`. */
+  readonly error?: string
+  readonly counts: CrawlCounts
+}
+
+export interface CrawlResource {
+  readonly url: string
+  /** The page that linked to it; null for the seed. */
+  readonly from: string | null
+  readonly status: number
+  readonly contentType: string | null
+  readonly bytes: number
+  readonly etag: string | null
+  readonly lastModified: string | null
+  readonly sha256: string | null
+  readonly fileId: string | null
+  /** True when the server answered a conditional request with 304; the other fields are those of the earlier crawl. */
+  readonly unchanged: boolean
+}
+
+export interface CrawlRefusal {
+  readonly url: string
+  readonly from: string | null
+  /** `scope`, `robots`, `depth`, `page_hook`, or the `untrustedFetch.get` reason (`address`, `too_large`, `rejected`, `status`, ...). */
+  readonly reason: string
+  readonly message: string
+}
+
+export interface CrawlSite {
+  /** Add to a worker surface's `jobs`. */
+  readonly job: Handler<Record<string, unknown>, unknown>
+  /** Job type of `job`. */
+  readonly jobType: string
+  /**
+   * Records a crawl and, unless the Blueprint's `crawl.requireApproval` is true, queues its job. Needs a Flow. Throws
+   * `CrawlSeedError` for a seed that is not http(s) or not inside its own scope. One audit record: who, seed, scope, limits.
+   */
+  start(options: CrawlStartOptions): Promise<{ id: string }>
+  /** Queues the job of a crawl that is `awaiting_approval`; one audit record. Throws `ConflictError` for a crawl in another state. */
+  approve(id: string): Promise<void>
+  status(id: string): Promise<CrawlStatus>
+  /** The listed resources, in the order they were found. */
+  resources(id: string): Promise<CrawlResource[]>
+  /** Every URL the crawl declined to fetch or could not, and every `onPage` failure, with the reason. */
+  refusals(id: string): Promise<CrawlRefusal[]>
+}
+
+export const DEFAULT_LIMITS: ResolvedCrawlLimits = {
+  perHostConcurrency: 1, minGapMs: 1000, maxPages: 1000, maxDepth: 10, maxBytes: 1024 ** 3, maxBytesPerResponse: 25 * 1024 * 1024,
+  maxWallMs: 3_600_000, requestTimeoutMs: 30_000, maxRedirects: 5, pageHookMs: 5000,
+}
+
+const NAME_RE = /^[A-Za-z][A-Za-z0-9_.-]*$/
+const ROBOTS_MAX_BYTES = 500 * 1024
+const ROBOTS_TRIES = 4
+/** Rules that allow nothing because the host's robots.txt could not be fetched for a reason that also bars the page itself. */
+interface BarredRules extends RobotsRules { readonly refusal: { reason: string; message: string } }
+const barred = (reason: string, message: string): BarredRules => ({ ...DISALLOW_ALL, refusal: { reason, message } })
+const HTML_TYPES = new Set(['text/html', 'application/xhtml+xml'])
+
+type StoredScope = { kind: 'host'; host: string } | { kind: 'domain'; domain: string }
+
+interface UrlRow { id: string; url: string; depth: number; fromUrl: string | null; redirects: number; seedChain: boolean }
+
+interface RunState {
+  stop: CrawlStoppedBy | null
+  inFlight: number
+  /** Per host: when the last request ended (started, with several in flight). */
+  last: Map<string, number>
+  hostInit: Map<string, Promise<void>>
+  robots: Map<string, Promise<RobotsRules | null>>
+}
+
+const num = (v: unknown): number => Number(v ?? 0)
+const str = (v: unknown): string | null => (v === null || v === undefined ? null : String(v))
+const baseType = (contentType: string | null): string => (contentType ?? '').split(';')[0]!.trim().toLowerCase()
+const describeError = (err: unknown): string => (err instanceof Error ? err.message : String(err)).slice(0, 500)
+
+function sleep(ms: number, signal: AbortSignal): Promise<void> {
+  return new Promise<void>(resolve => {
+    if (ms <= 0 || signal.aborted) { resolve(); return }
+    const done = (): void => { clearTimeout(timer); signal.removeEventListener('abort', done); resolve() }
+    const timer = setTimeout(done, ms)
+    signal.addEventListener('abort', done, { once: true })
+  })
+}
+
+function inScope(scope: StoredScope, hostname: string): boolean {
+  if (scope.kind === 'host') return hostname === scope.host
+  return hostname === scope.domain || hostname.endsWith('.' + scope.domain)
+}
+
+function resolveLimits(given: CrawlLimits | undefined): ResolvedCrawlLimits {
+  const out: Record<string, number> = { ...DEFAULT_LIMITS }
+  for (const [key, value] of Object.entries(given ?? {})) {
+    if (!(key in DEFAULT_LIMITS)) throw new CrawlSeedError(`unknown limit "${key}" (limits: ${Object.keys(DEFAULT_LIMITS).join(', ')}).`)
+    if (value === undefined) continue
+    const min = key === 'minGapMs' || key === 'maxRedirects' || key === 'maxDepth' ? 0 : 1
+    if (typeof value !== 'number' || !Number.isFinite(value) || value < min) throw new CrawlSeedError(`limit ${key} must be a number of at least ${String(min)}, got ${String(value)}.`)
+    out[key] = value
+  }
+  return out as unknown as ResolvedCrawlLimits
+}
+
+function parseSeed(seed: string, scope: CrawlScope | undefined): { url: URL; stored: StoredScope } {
+  let url: URL
+  try { url = new URL(seed) } catch { throw new CrawlSeedError(`the seed "${seed}" is not a URL.`) }
+  if (url.protocol !== 'http:' && url.protocol !== 'https:') throw new CrawlSeedError(`the seed must be an http or https URL, got ${url.protocol}`)
+  url.hash = ''
+  let stored: StoredScope
+  if (scope === undefined || scope === 'host') {
+    stored = { kind: 'host', host: url.hostname }
+  } else if (typeof scope === 'object' && typeof scope.domain === 'string' && scope.domain.trim() !== '') {
+    stored = { kind: 'domain', domain: scope.domain.trim().toLowerCase().replace(/^\.+|\.+$/g, '') }
+  } else {
+    throw new CrawlSeedError(`scope must be 'host' or { domain: 'example.gov.uk' }.`)
+  }
+  if (!inScope(stored, url.hostname)) {
+    throw new CrawlSeedError(`the seed's host ${url.hostname} is not inside its scope (${stored.kind === 'host' ? stored.host : stored.domain}).`)
+  }
+  return { url, stored }
+}
+
+/** Creates a site crawler; see `crawl.create`. */
+export function createCrawl(config: CrawlConfig): CrawlSite {
+  const blueprint = config.instance.config.raw.crawl
+  if (blueprint === undefined) throw new CrawlNotEnabledError()
+  if (typeof config.name !== 'string' || !NAME_RE.test(config.name)) {
+    throw new Error(`[pipework] crawl.create: name "${String(config.name)}" must start with a letter and use letters, digits, "_", "." or "-".\n`)
+  }
+  if (typeof config.userAgent !== 'string' || config.userAgent.trim() === '') throw new Error('[pipework] crawl.create: userAgent is required: sites read it to decide what you may fetch.\n')
+  const requireApproval = blueprint.requireApproval === true
+  const jobType = `pipework.crawl.${config.name}`
+  const database = config.database
+  const db = (): DB => pipeSystem(database)
+  const robotsRetryMs = config.robotsRetryMs ?? 2000
+  const urlIsSafeToTrace = config.traceUrls !== false
+
+  const enqueue = async (id: string): Promise<void> => {
+    await config.queue.enqueue(db(), { type: jobType, payload: { crawlId: id }, liveKey: id, continueFlow: true, maxAttempts: config.maxAttempts ?? 5 })
+  }
+
+  const startStep = __flowStep({ ref: 'pipework/crawl:start', capture: [null] }, async (options: CrawlStartOptions): Promise<{ id: string }> => {
+    const { url, stored } = parseSeed(options.seed, options.scope)
+    const limits = resolveLimits(options.limits)
+    const flow = getContext()
+    const actor = flow === undefined ? null : resolveActorId(flow)
+    const state: CrawlState = requireApproval ? 'awaiting_approval' : 'running'
+    const rows = await db().execute(sql`
+      INSERT INTO pipework_crawl (site, seed, scope, limits, state, tenant_id, started_by, run_started_at)
+      VALUES (${config.name}, ${url.href}, ${JSON.stringify(stored)}::jsonb, ${JSON.stringify(limits)}::jsonb, ${state},
+              ${flow?.tenant ?? null}, ${actor}, ${requireApproval ? null : sql`now()`})
+      RETURNING id::text AS id`)
+    const id = rows[0]!['id'] as string
+    await db().execute(sql`INSERT INTO pipework_crawl_url (crawl_id, url, depth, seed_chain) VALUES (${id}::uuid, ${url.href}, 0, true)`)
+    await auditCrawl('start', id, { site: config.name, seed: url.href, scope: stored, limits, awaitingApproval: requireApproval }, database)
+    if (!requireApproval) await enqueue(id)
+    return { id }
+  })
+
+  const approveStep = __flowStep({ ref: 'pipework/crawl:approve', capture: [null] }, async (id: string): Promise<void> => {
+    await requireCrawl(id)
+    const flow = getContext()
+    const actor = flow === undefined ? null : resolveActorId(flow)
+    const rows = await db().execute(sql`
+      UPDATE pipework_crawl SET state = 'running', approved_by = ${actor}, run_started_at = now()
+       WHERE id = ${id}::uuid AND site = ${config.name} AND state = 'awaiting_approval' RETURNING seed`)
+    if (rows.length === 0) {
+      const found = await db().execute(sql`SELECT state FROM pipework_crawl WHERE id = ${id}::uuid AND site = ${config.name}`)
+      if (found.length === 0) throw new NotFoundError(`Crawl ${id}`)
+      throw new ConflictError(`Crawl ${id} is not awaiting approval (it is ${String(found[0]!['state'])}).`)
+    }
+    await auditCrawl('approve', id, { site: config.name, seed: rows[0]!['seed'] }, database)
+    await enqueue(id)
+  })
+
+  async function loadCrawl(id: string): Promise<{ state: CrawlState; limits: ResolvedCrawlLimits; scope: StoredScope; seed: string; runStartedAt: number; tenantId: string | null; stoppedBy: CrawlStoppedBy | null; error: string | null; bytes: number } | null> {
+    const rows = await db().execute(sql`
+      SELECT state, limits, scope, seed, tenant_id, stopped_by, error, bytes::float8 AS bytes,
+             (extract(epoch from run_started_at) * 1000)::float8 AS run_started_ms
+        FROM pipework_crawl WHERE id = ${id}::uuid AND site = ${config.name}`)
+    const r = rows[0]
+    if (r === undefined) return null
+    return {
+      state: r['state'] as CrawlState, limits: r['limits'] as ResolvedCrawlLimits, scope: r['scope'] as StoredScope, seed: r['seed'] as string,
+      runStartedAt: num(r['run_started_ms']), tenantId: str(r['tenant_id']),
+      stoppedBy: str(r['stopped_by']) as CrawlStoppedBy | null, error: str(r['error']), bytes: num(r['bytes']),
+    }
+  }
+
+  /** The crawl, or NotFoundError: also when it belongs to another tenant than the Flow's. */
+  const requireCrawl = async (id: string): Promise<NonNullable<Awaited<ReturnType<typeof loadCrawl>>>> => {
+    const crawl = await loadCrawl(id)
+    const tenant = getContext()?.tenant ?? null
+    if (crawl === null || (crawl.tenantId !== null && crawl.tenantId !== tenant)) throw new NotFoundError(`Crawl ${id}`)
+    return crawl
+  }
+
+  const status = async (id: string): Promise<CrawlStatus> => {
+    const crawl = await requireCrawl(id)
+    const counted = await db().execute(sql`
+      SELECT count(*) FILTER (WHERE state = 'queued')::int AS queued, count(*) FILTER (WHERE state = 'fetching')::int AS fetching,
+             count(*) FILTER (WHERE state = 'done')::int AS done, count(*) FILTER (WHERE state = 'refused')::int AS refused
+        FROM pipework_crawl_url WHERE crawl_id = ${id}::uuid`)
+    const listed = await db().execute(sql`
+      SELECT count(*)::int AS listed, count(*) FILTER (WHERE unchanged)::int AS unchanged FROM pipework_crawl_resource WHERE crawl_id = ${id}::uuid`)
+    const c = counted[0]!
+    return {
+      state: crawl.state,
+      scope: crawl.scope.kind === 'host' ? { host: crawl.scope.host } : { domain: crawl.scope.domain },
+      ...(crawl.stoppedBy !== null ? { stoppedBy: crawl.stoppedBy } : {}),
+      ...(crawl.error !== null ? { error: crawl.error } : {}),
+      counts: {
+        queued: num(c['queued']), fetching: num(c['fetching']), done: num(c['done']), refused: num(c['refused']),
+        listed: num(listed[0]!['listed']), unchanged: num(listed[0]!['unchanged']), bytes: crawl.bytes,
+      },
+    }
+  }
+
+  const resources = async (id: string): Promise<CrawlResource[]> => {
+    await requireCrawl(id)
+    const rows = await db().execute(sql`
+      SELECT r.url, r.from_url, r.status, r.content_type, r.bytes::float8 AS bytes, r.etag, r.last_modified, r.sha256, r.file_id, r.unchanged
+        FROM pipework_crawl_resource r JOIN pipework_crawl c ON c.id = r.crawl_id
+       WHERE r.crawl_id = ${id}::uuid AND c.site = ${config.name} ORDER BY r.created_at, r.id`)
+    return rows.map(r => ({
+      url: r['url'] as string, from: str(r['from_url']), status: num(r['status']), contentType: str(r['content_type']), bytes: num(r['bytes']),
+      etag: str(r['etag']), lastModified: str(r['last_modified']), sha256: str(r['sha256']), fileId: str(r['file_id']), unchanged: r['unchanged'] === true,
+    }))
+  }
+
+  const refusals = async (id: string): Promise<CrawlRefusal[]> => {
+    await requireCrawl(id)
+    const rows = await db().execute(sql`
+      SELECT u.url, u.from_url, u.reason, u.message
+        FROM pipework_crawl_url u JOIN pipework_crawl c ON c.id = u.crawl_id
+       WHERE u.crawl_id = ${id}::uuid AND c.site = ${config.name} AND (u.state = 'refused' OR u.reason = 'page_hook') ORDER BY u.seq`)
+    return rows.map(r => ({ url: r['url'] as string, from: str(r['from_url']), reason: str(r['reason']) ?? '', message: str(r['message']) ?? '' }))
+  }
+
+  // ---- the job ----
+
+  async function run(crawlId: string, signal: AbortSignal): Promise<void> {
+    const crawl = await loadCrawl(crawlId)
+    if (crawl === null || crawl.state !== 'running') return
+    const { limits } = crawl
+    let scope = crawl.scope
+    const state: RunState = { stop: null, inFlight: 0, last: new Map(), hostInit: new Map(), robots: new Map() }
+    // Only one job runs a crawl, so a row still 'fetching' is the one that was in flight when the last attempt died.
+    await db().execute(sql`UPDATE pipework_crawl_url SET state = 'queued' WHERE crawl_id = ${crawlId}::uuid AND state = 'fetching'`)
+
+    const hostRow = async (host: string): Promise<{ robotsState: string | null; robotsText: string | null; attempts: number; lastMs: number }> => {
+      await db().execute(sql`INSERT INTO pipework_crawl_host (crawl_id, host) VALUES (${crawlId}::uuid, ${host}) ON CONFLICT (crawl_id, host) DO NOTHING`)
+      const r = (await db().execute(sql`
+        SELECT robots_state, robots_text, robots_attempts, last_request_at_ms::float8 AS last_ms FROM pipework_crawl_host WHERE crawl_id = ${crawlId}::uuid AND host = ${host}`))[0]!
+      return { robotsState: str(r['robots_state']), robotsText: str(r['robots_text']), attempts: num(r['robots_attempts']), lastMs: num(r['last_ms']) }
+    }
+
+    const gapFor = (rules: RobotsRules | null): number => Math.max(limits.minGapMs, (rules?.crawlDelaySeconds ?? 0) * 1000)
+
+    /** Waits for this host's turn, then returns a function that records the request's end. A request starts at least `gap` after the last one ended (started, with several in flight). */
+    const takeTurn = async (host: string, gap: number): Promise<() => Promise<void>> => {
+      let init = state.hostInit.get(host)
+      if (init === undefined) {
+        init = hostRow(host).then(row => { state.last.set(host, row.lastMs) })
+        state.hostInit.set(host, init)
+      }
+      await init
+      const start = Math.max(Date.now(), state.last.get(host)! + gap)
+      if (limits.perHostConcurrency > 1) state.last.set(host, start)
+      await sleep(start - Date.now(), signal)
+      return async () => {
+        const end = Date.now()
+        state.last.set(host, Math.max(state.last.get(host)!, end))
+        await db().execute(sql`UPDATE pipework_crawl_host SET last_request_at_ms = ${end} WHERE crawl_id = ${crawlId}::uuid AND host = ${host}`)
+      }
+    }
+
+    const ua = { 'user-agent': config.userAgent }
+    const network = {
+      ...(config.network?.resolve !== undefined ? { resolve: config.network.resolve } : {}),
+      ...(config.network?.connect !== undefined ? { connect: config.network.connect } : {}),
+    }
+
+    /** The host's robots rules, fetched once before any other request to it; null when it stayed unreachable (the crawl stops). */
+    const robotsFor = (host: string, protocol: string): Promise<RobotsRules | null> => {
+      let pending = state.robots.get(host)
+      if (pending === undefined) {
+        pending = loadRobots(host, protocol)
+        state.robots.set(host, pending)
+      }
+      return pending
+    }
+
+    const loadRobots = async (host: string, protocol: string): Promise<RobotsRules | null> => {
+      const row = await hostRow(host)
+      if (row.robotsState === 'ok') return parseRobots(row.robotsText ?? '', config.userAgent)
+      if (row.robotsState === 'allow_all') return ALLOW_ALL
+      if (row.robotsState === 'disallow_all') {
+        const kept = JSON.parse(row.robotsText ?? 'null') as { reason: string; message: string } | null
+        return kept === null ? DISALLOW_ALL : barred(kept.reason, kept.message)
+      }
+      const settle = async (robotsState: string, text: string | null): Promise<void> => {
+        await db().execute(sql`UPDATE pipework_crawl_host SET robots_state = ${robotsState}, robots_text = ${text} WHERE crawl_id = ${crawlId}::uuid AND host = ${host}`)
+      }
+      for (let attempt = row.attempts; attempt < ROBOTS_TRIES; attempt++) {
+        if (signal.aborted) return null
+        if (attempt > 0) await sleep(robotsRetryMs * 2 ** (attempt - 1), signal)
+        const done = await takeTurn(host, limits.minGapMs)
+        const got = await untrustedGet(`${protocol}//${host}/robots.txt`, {
+          maxBytes: ROBOTS_MAX_BYTES, timeoutMs: limits.requestTimeoutMs, maxRedirects: limits.maxRedirects, headers: ua, urlIsSafeToTrace,
+          followRedirect: (to) => new URL(to).host === host ? true : 'robots.txt redirects to another host',
+          ...network,
+        })
+        await done()
+        if (got.ok) {
+          const text = new TextDecoder().decode(got.body)
+          await settle('ok', text)
+          return parseRobots(text, config.userAgent)
+        }
+        if (got.reason === 'status' && got.status !== undefined && got.status >= 400 && got.status < 500) {
+          await settle('allow_all', null)
+          return ALLOW_ALL
+        }
+        if (got.reason === 'status' || got.reason === 'network' || got.reason === 'timeout') {
+          await db().execute(sql`UPDATE pipework_crawl_host SET robots_attempts = ${attempt + 1} WHERE crawl_id = ${crawlId}::uuid AND host = ${host}`)
+          continue
+        }
+        const refusal = { reason: got.reason, message: got.message }
+        await settle('disallow_all', JSON.stringify(refusal))
+        return barred(refusal.reason, refusal.message)
+      }
+      return signal.aborted ? null : (state.stop = 'robots_unreachable', null)
+    }
+
+    const insertLinks = async (tx: DB, rows: Array<{ url: string; depth: number; from: string; state: string; reason: string | null; message: string | null; redirects: number; seedChain?: boolean }>): Promise<void> => {
+      if (rows.length === 0) return
+      await tx.execute(sql`
+        INSERT INTO pipework_crawl_url (crawl_id, url, depth, from_url, redirects, state, reason, message, seed_chain)
+        SELECT ${crawlId}::uuid, x.url, x.depth, x.from_url, x.redirects, x.state, x.reason, x.message, x.seed_chain
+          FROM jsonb_to_recordset(${JSON.stringify(rows.map(r => ({ url: r.url, depth: r.depth, from_url: r.from, redirects: r.redirects, state: r.state, reason: r.reason, message: r.message, seed_chain: r.seedChain === true })))}::jsonb)
+               AS x(url text, depth int, from_url text, redirects int, state text, reason text, message text, seed_chain boolean)
+        ON CONFLICT (crawl_id, url) DO NOTHING`)
+    }
+
+    const finishUrl = async (row: UrlRow, outcome: { state: 'done' | 'refused'; reason?: string; message?: string; status?: number | null; requested?: boolean }, extra?: (tx: DB) => Promise<void>): Promise<void> => {
+      await db().transaction(async (tx) => {
+        await extra?.(tx)
+        await tx.execute(sql`
+          UPDATE pipework_crawl_url SET state = ${outcome.state}, reason = ${outcome.reason ?? null}, message = ${outcome.message ?? null}, status = ${outcome.status ?? null}, requested = requested OR ${outcome.requested === true}
+           WHERE id = ${row.id}::uuid`)
+      })
+    }
+
+    const runHook = async (url: string, html: string): Promise<string | null> => {
+      const hook = config.onPage
+      if (hook === undefined) return null
+      const controller = new AbortController()
+      let timer: ReturnType<typeof setTimeout> | undefined
+      const overrun = new Promise<never>((_, reject) => {
+        timer = setTimeout(() => {
+          const err = new Error(`onPage took longer than ${String(limits.pageHookMs)} ms.`)
+          controller.abort(err)
+          reject(err)
+        }, limits.pageHookMs)
+      })
+      try {
+        const work = Promise.resolve().then(() => hook({ url, html, signal: controller.signal }))
+        work.catch(() => undefined)
+        await Promise.race([work, overrun])
+        return null
+      } catch (err) {
+        return describeError(err)
+      } finally {
+        clearTimeout(timer)
+      }
+    }
+
+    const previous = async (url: string): Promise<{ etag: string | null; lastModified: string | null; status: number; contentType: string | null; bytes: number; sha256: string | null; fileId: string | null } | null> => {
+      const rows = await db().execute(sql`
+        SELECT r.etag, r.last_modified, r.status, r.content_type, r.bytes::float8 AS bytes, r.sha256, r.file_id
+          FROM pipework_crawl_resource r JOIN pipework_crawl c ON c.id = r.crawl_id
+         WHERE c.site = ${config.name} AND c.seed = ${crawl.seed} AND c.id <> ${crawlId}::uuid AND c.created_at < (SELECT created_at FROM pipework_crawl WHERE id = ${crawlId}::uuid)
+           AND r.url = ${url} AND (r.etag IS NOT NULL OR r.last_modified IS NOT NULL)
+         ORDER BY c.created_at DESC LIMIT 1`)
+      const r = rows[0]
+      if (r === undefined) return null
+      return { etag: str(r['etag']), lastModified: str(r['last_modified']), status: num(r['status']), contentType: str(r['content_type']), bytes: num(r['bytes']), sha256: str(r['sha256']), fileId: str(r['file_id']) }
+    }
+
+    const movesScope = (row: UrlRow, to: URL): boolean => scope.kind === 'host' && row.seedChain && row.depth === 0 && !inScope(scope, to.hostname)
+
+    const processUrl = async (row: UrlRow): Promise<void> => {
+      const target = new URL(row.url)
+      const rules = await robotsFor(target.host, target.protocol)
+      if (rules === null) {
+        if (!signal.aborted) await db().execute(sql`UPDATE pipework_crawl_url SET state = 'queued' WHERE id = ${row.id}::uuid`)
+        return
+      }
+      if (!rules.allows(target.pathname + target.search)) {
+        const why = (rules as Partial<BarredRules>).refusal
+        await finishUrl(row, { state: 'refused', reason: why?.reason ?? 'robots', message: why?.message ?? 'robots.txt does not allow this path.' })
+        return
+      }
+      const done = await takeTurn(target.host, gapFor(rules))
+      if (signal.aborted) {
+        await db().execute(sql`UPDATE pipework_crawl_url SET state = 'queued' WHERE id = ${row.id}::uuid`)
+        return
+      }
+      const prior = await previous(row.url)
+      const headers: Record<string, string> = { ...ua }
+      if (prior?.etag != null) headers['if-none-match'] = prior.etag
+      if (prior?.lastModified != null) headers['if-modified-since'] = prior.lastModified
+      const hop: { to: string | null } = { to: null }
+      const got = await untrustedGet(row.url, {
+        maxBytes: limits.maxBytesPerResponse, timeoutMs: limits.requestTimeoutMs, maxRedirects: limits.maxRedirects, headers, urlIsSafeToTrace,
+        // A redirect is not followed inside the request: its target is a URL of the crawl like any link, so it meets scope, robots and the host's gap.
+        followRedirect: (to) => { hop.to = to; return 'The crawler handles redirects itself.' },
+        accept: (head) => {
+          const type = baseType(head.contentType)
+          if (HTML_TYPES.has(type)) return true
+          return config.accept({ url: row.url, contentType: type }) || 'This is neither a page nor a file the app lists.'
+        },
+        ...network,
+      })
+      await done()
+
+      if (!got.ok) {
+        if (got.reason === 'rejected' && hop.to !== null) {
+          const hops = row.redirects + 1
+          let to: URL | null = null
+          try { to = new URL(hop.to) } catch { /* refused below */ }
+          if (hops > limits.maxRedirects) await finishUrl(row, { requested: true, state: 'refused', reason: 'redirects', message: 'The link redirects too many times.' })
+          else if (to === null || (!inScope(scope, to.hostname) && !movesScope(row, to))) await finishUrl(row, { requested: true, state: 'refused', reason: 'scope', message: 'The link redirects to a page outside the crawl\'s scope.' })
+          else {
+            to.hash = ''
+            // The seed's own redirect chain may move a host scope (example.gov.uk to www.example.gov.uk).
+            const moved: StoredScope | null = inScope(scope, to.hostname) ? null : { kind: 'host', host: to.hostname }
+            await finishUrl(row, { requested: true, state: 'done', reason: 'redirected', message: 'Redirected; the target is queued.' }, async (tx) => {
+              if (moved !== null) await tx.execute(sql`UPDATE pipework_crawl SET scope = ${JSON.stringify(moved)}::jsonb WHERE id = ${crawlId}::uuid`)
+              await insertLinks(tx, [{ url: to.href, depth: row.depth, from: row.fromUrl ?? row.url, state: 'queued', reason: null, message: null, redirects: hops, seedChain: row.seedChain }])
+            })
+            if (moved !== null) scope = moved
+          }
+          return
+        }
+        if (got.reason === 'status' && got.status === 304 && prior !== null) {
+          await finishUrl(row, { requested: true, state: 'done', status: 304 }, tx => tx.execute(sql`
+            INSERT INTO pipework_crawl_resource (crawl_id, url, from_url, status, content_type, bytes, etag, last_modified, sha256, file_id, unchanged)
+            VALUES (${crawlId}::uuid, ${row.url}, ${row.fromUrl}, ${prior.status}, ${prior.contentType}, ${prior.bytes}, ${prior.etag}, ${prior.lastModified}, ${prior.sha256}, ${prior.fileId}, true)
+            ON CONFLICT (crawl_id, url) DO NOTHING`).then(() => undefined))
+          return
+        }
+        await finishUrl(row, { requested: true, state: 'refused', reason: got.reason, message: got.message, status: got.status ?? null })
+        return
+      }
+
+      const type = baseType(got.contentType)
+      const isHtml = HTML_TYPES.has(type)
+      const listed = config.accept({ url: got.url, contentType: type })
+      let hookFault: string | null = null
+      let links: string[] = []
+      if (isHtml) {
+        const html = new TextDecoder().decode(got.body)
+        links = extractLinks(html, got.url)
+        hookFault = await runHook(row.url, html)
+      }
+      let sha256: string | null = null
+      let fileId: string | null = null
+      if (listed && config.store !== undefined) {
+        const file = await config.store.put({ body: Readable.from(Buffer.from(got.body)), mediaType: type === '' ? 'application/octet-stream' : type, metadata: { url: got.url, crawl: crawlId } })
+        sha256 = file.sha256
+        fileId = file.id
+      }
+      const children = links.map(link => {
+        const u = new URL(link)
+        const depth = row.depth + 1
+        if (!inScope(scope, u.hostname)) return { url: link, depth, from: row.url, state: 'refused', reason: 'scope', message: 'The link points outside the crawl\'s scope.', redirects: 0 }
+        if (depth > limits.maxDepth) return { url: link, depth, from: row.url, state: 'refused', reason: 'depth', message: `The link is more than ${String(limits.maxDepth)} links from the seed.`, redirects: 0 }
+        return { url: link, depth, from: row.url, state: 'queued', reason: null, message: null, redirects: 0 }
+      })
+      await finishUrl(
+        row,
+        hookFault === null ? { requested: true, state: 'done', status: got.status } : { requested: true, state: 'done', status: got.status, reason: 'page_hook', message: hookFault },
+        async (tx) => {
+          await tx.execute(sql`UPDATE pipework_crawl SET bytes = bytes + ${got.bytes} WHERE id = ${crawlId}::uuid`)
+          if (listed) {
+            await tx.execute(sql`
+              INSERT INTO pipework_crawl_resource (crawl_id, url, from_url, status, content_type, bytes, etag, last_modified, sha256, file_id)
+              VALUES (${crawlId}::uuid, ${row.url}, ${row.fromUrl}, ${got.status}, ${type === '' ? null : type}, ${got.bytes}, ${got.etag}, ${got.lastModified}, ${sha256}, ${fileId})
+              ON CONFLICT (crawl_id, url) DO NOTHING`)
+          }
+          await insertLinks(tx, children)
+        },
+      )
+    }
+
+    /** The next queued URL, claimed so no other lane takes it; breadth first. */
+    const claim = async (): Promise<UrlRow | null> => {
+      const rows = await db().execute(sql`
+        UPDATE pipework_crawl_url SET state = 'fetching'
+         WHERE id = (SELECT id FROM pipework_crawl_url WHERE crawl_id = ${crawlId}::uuid AND state = 'queued'
+                      ORDER BY depth, seq LIMIT 1 FOR UPDATE SKIP LOCKED)
+        RETURNING id::text AS id, url, depth, from_url, redirects, seed_chain`)
+      const r = rows[0]
+      return r === undefined ? null : { id: r['id'] as string, url: r['url'] as string, depth: num(r['depth']), fromUrl: str(r['from_url']), redirects: num(r['redirects']), seedChain: r['seed_chain'] === true }
+    }
+
+    const limitHit = async (): Promise<CrawlStoppedBy | null> => {
+      if (Date.now() - crawl.runStartedAt > limits.maxWallMs) return 'wall'
+      const r = (await db().execute(sql`
+        SELECT (SELECT count(*) FROM pipework_crawl_url WHERE crawl_id = ${crawlId}::uuid AND requested)::int AS pages,
+               (SELECT bytes::float8 FROM pipework_crawl WHERE id = ${crawlId}::uuid) AS bytes`))[0]!
+      if (num(r['pages']) >= limits.maxPages) return 'pages'
+      if (num(r['bytes']) >= limits.maxBytes) return 'bytes'
+      return null
+    }
+
+    const lane = async (): Promise<void> => {
+      while (!signal.aborted && state.stop === null) {
+        const hit = await limitHit()
+        if (hit !== null) {
+          const queued = hit === 'wall' || hit === 'bytes' || num((await db().execute(sql`SELECT count(*)::int AS n FROM pipework_crawl_url WHERE crawl_id = ${crawlId}::uuid AND state = 'queued'`))[0]!['n']) > 0
+          if (queued) state.stop = hit
+          else if (state.inFlight === 0) return
+        }
+        if (state.stop !== null) return
+        const row = await claim()
+        if (row === null) {
+          if (state.inFlight === 0) return
+          await sleep(25, signal)
+          continue
+        }
+        state.inFlight++
+        try { await processUrl(row) } finally { state.inFlight-- }
+      }
+    }
+
+    await Promise.all(Array.from({ length: limits.perHostConcurrency }, () => lane()))
+    if (signal.aborted) return
+    if (state.stop !== null) {
+      await db().execute(sql`UPDATE pipework_crawl SET state = 'stopped', stopped_by = ${state.stop}, finished_at = now() WHERE id = ${crawlId}::uuid`)
+      return
+    }
+    const depthCut = await db().execute(sql`SELECT 1 FROM pipework_crawl_url WHERE crawl_id = ${crawlId}::uuid AND reason = 'depth' LIMIT 1`)
+    if (depthCut.length > 0) {
+      await db().execute(sql`UPDATE pipework_crawl SET state = 'stopped', stopped_by = 'depth', finished_at = now() WHERE id = ${crawlId}::uuid`)
+      return
+    }
+    await db().execute(sql`UPDATE pipework_crawl SET state = 'done', finished_at = now() WHERE id = ${crawlId}::uuid`)
+  }
+
+  const builder = database !== undefined ? fitting.use(database) : fitting.use()
+  const job = builder.job(jobType)
+    .transaction('manual')
+    .input(schema.check.object({ crawlId: schema.check.string() }))
+    .fit((async ({ input, signal }: { input: { crawlId: string }; signal: AbortSignal }) => {
+      try {
+        await run(input.crawlId, signal)
+      } catch (err) {
+        const flow = getContext()
+        const last = flow?.jobAttempt != null && flow.jobMaxAttempts != null && flow.jobAttempt >= flow.jobMaxAttempts
+        if (last) {
+          getBaseLogger().error({ err, crawlId: input.crawlId }, 'Crawl job failed on its last attempt')
+          await db().execute(sql`UPDATE pipework_crawl SET state = 'failed', error = ${describeError(err)}, finished_at = now() WHERE id = ${input.crawlId}::uuid`)
+        }
+        throw err
+      }
+    }) as never) as unknown as Handler<Record<string, unknown>, unknown>
+
+  return {
+    job,
+    jobType,
+    start: (options) => startStep(options),
+    approve: (id) => approveStep(id),
+    status, resources, refusals,
+  }
+}