pipework 0.252.0 → 0.253.0

package/src/infra/crawl/index.ts

@@ -0,0 +1,9 @@
+export { createCrawl, DEFAULT_LIMITS } from './handle.js'
+export type {
+  CrawlConfig, CrawlSite, CrawlStartOptions, CrawlLimits, ResolvedCrawlLimits, CrawlScope, CrawlState, CrawlStoppedBy,
+  CrawlStatus, CrawlCounts, CrawlResource, CrawlRefusal, CrawlPage,
+} from './handle.js'
+export { CrawlNotEnabledError, CrawlSeedError } from './errors.js'
+export { parseRobots, productToken, ALLOW_ALL, DISALLOW_ALL } from './robots.js'
+export type { RobotsRules } from './robots.js'
+export { extractLinks } from './links.js'

package/src/infra/crawl/links.ts

@@ -0,0 +1,160 @@
+// Link extraction from untrusted HTML: a single forward scan, no DOM, nothing executed, no backtracking patterns.
+// The caller bounds the input size; the scan is linear in it apart from the search for a raw-text element's end.
+
+const RAW_TEXT = new Set(['script', 'style', 'template'])
+const HREF_TAGS = new Set(['a', 'area', 'link'])
+const SRC_TAGS = new Set(['iframe', 'frame'])
+
+const NAMED: Record<string, string> = { amp: '&', quot: '"', apos: "'", lt: '<', gt: '>' }
+
+/** Decodes `&amp;` `&quot;` `&apos;` `&lt;` `&gt;` `&#NN;` and `&#xNN;`; anything else is left as written. */
+export function decodeEntities(value: string): string {
+  let at = value.indexOf('&')
+  if (at === -1) return value
+  let out = ''
+  let from = 0
+  while (at !== -1) {
+    const semi = value.indexOf(';', at + 1)
+    let replaced: string | null = null
+    if (semi !== -1 && semi - at <= 10) {
+      const body = value.slice(at + 1, semi)
+      if (body[0] === '#') {
+        const hex = body[1] === 'x' || body[1] === 'X'
+        const digits = body.slice(hex ? 2 : 1)
+        if (digits !== '' && (hex ? /^[0-9a-fA-F]+$/ : /^[0-9]+$/).test(digits)) {
+          const code = parseInt(digits, hex ? 16 : 10)
+          if (code > 0 && code <= 0x10ffff && !(code >= 0xd800 && code <= 0xdfff)) replaced = String.fromCodePoint(code)
+        }
+      } else {
+        replaced = NAMED[body] ?? null
+      }
+    }
+    if (replaced === null) {
+      out += value.slice(from, at + 1)
+      from = at + 1
+    } else {
+      out += value.slice(from, at) + replaced
+      from = semi + 1
+    }
+    at = value.indexOf('&', from)
+  }
+  return out + value.slice(from)
+}
+
+function isSpace(c: string): boolean {
+  return c === ' ' || c === '\t' || c === '\n' || c === '\r' || c === '\f'
+}
+
+/** Index of `needle` (lower-case) in `html` at or after `from`, ignoring the case of `html`; -1 when absent. */
+function indexOfIgnoreCase(html: string, needle: string, from: number): number {
+  const first = needle.charCodeAt(0)
+  const firstUpper = needle.toUpperCase().charCodeAt(0)
+  const last = html.length - needle.length
+  for (let i = from; i <= last; i++) {
+    const c = html.charCodeAt(i)
+    if (c !== first && c !== firstUpper) continue
+    if (html.slice(i, i + needle.length).toLowerCase() === needle) return i
+  }
+  return -1
+}
+
+interface Tag { name: string; attrs: Map<string, string>; end: number; closed: boolean }
+
+/** Reads a start tag whose `<` is at `start`; `end` is the index after its `>`; `closed` is false when the input ended first. */
+function readTag(html: string, start: number): Tag | null {
+  let i = start + 1
+  const n = html.length
+  const nameStart = i
+  if (!/[A-Za-z]/.test(html[i] ?? '')) return null
+  while (i < n && /[A-Za-z0-9:-]/.test(html[i]!)) i++
+  if (i === nameStart) return null
+  const name = html.slice(nameStart, i).toLowerCase()
+  const attrs = new Map<string, string>()
+  while (i < n) {
+    while (i < n && (isSpace(html[i]!) || html[i] === '/')) i++
+    if (i >= n) break
+    if (html[i] === '>') { i++; return { name, attrs, end: i, closed: true } }
+    const attrStart = i
+    while (i < n && !isSpace(html[i]!) && html[i] !== '=' && html[i] !== '>' && html[i] !== '/') i++
+    const attr = html.slice(attrStart, i).toLowerCase()
+    while (i < n && isSpace(html[i]!)) i++
+    let value = ''
+    if (html[i] === '=') {
+      i++
+      while (i < n && isSpace(html[i]!)) i++
+      const q = html[i]
+      if (q === '"' || q === "'") {
+        const close = html.indexOf(q, i + 1)
+        const stop = close === -1 ? n : close
+        value = html.slice(i + 1, stop)
+        i = close === -1 ? n : close + 1
+      } else {
+        const valueStart = i
+        while (i < n && !isSpace(html[i]!) && html[i] !== '>') i++
+        value = html.slice(valueStart, i)
+      }
+    }
+    if (attr !== '' && !attrs.has(attr)) attrs.set(attr, decodeEntities(value))
+  }
+  return { name, attrs, end: n, closed: false }
+}
+
+/**
+ * The http(s) URLs `html` links to, resolved against `pageUrl` (or the page's first `<base href>`), without fragments,
+ * each once, in document order. Reads `href` of `a`, `area` and `link`, and `src` of `iframe` and `frame`; skips comments
+ * and the bodies of `script`, `style` and `template`.
+ */
+export function extractLinks(html: string, pageUrl: string): string[] {
+  const raw: string[] = []
+  let baseHref: string | null = null
+  const n = html.length
+  let i = 0
+  while (i < n) {
+    const lt = html.indexOf('<', i)
+    if (lt === -1) break
+    if (html.startsWith('<!--', lt)) {
+      const close = html.indexOf('-->', lt + 4)
+      i = close === -1 ? n : close + 3
+      continue
+    }
+    const next = html[lt + 1]
+    if (next === '!' || next === '?' || next === '/') {
+      const close = html.indexOf('>', lt + 2)
+      i = close === -1 ? n : close + 1
+      continue
+    }
+    const tag = readTag(html, lt)
+    if (tag === null) { i = lt + 1; continue }
+    i = tag.end
+    if (!tag.closed) break
+    if (RAW_TEXT.has(tag.name)) {
+      const close = indexOfIgnoreCase(html, `</${tag.name}`, i)
+      i = close === -1 ? n : close
+      continue
+    }
+    if (tag.name === 'base') {
+      const href = tag.attrs.get('href')
+      if (baseHref === null && href !== undefined) baseHref = href
+    } else if (HREF_TAGS.has(tag.name)) {
+      const href = tag.attrs.get('href')
+      if (href !== undefined) raw.push(href)
+    } else if (SRC_TAGS.has(tag.name)) {
+      const src = tag.attrs.get('src')
+      if (src !== undefined) raw.push(src)
+    }
+  }
+  let base = pageUrl
+  if (baseHref !== null) {
+    try { base = new URL(baseHref.trim(), pageUrl).href } catch { /* an unusable base is ignored */ }
+  }
+  const seen = new Set<string>()
+  const out: string[] = []
+  for (const value of raw) {
+    let url: URL
+    try { url = new URL(value.trim(), base) } catch { continue }
+    if (url.protocol !== 'http:' && url.protocol !== 'https:') continue
+    url.hash = ''
+    if (!seen.has(url.href)) { seen.add(url.href); out.push(url.href) }
+  }
+  return out
+}

package/src/infra/crawl/namespace.ts

@@ -0,0 +1,36 @@
+import { createCrawl } from './handle.js'
+
+/** Crawl — a polite, sandboxed site crawler that runs as a durable job. */
+export const crawl = {
+  /**
+   * Creates a site crawler at boot and returns its handle: `job` (add to a worker surface's `jobs`), `start`, `approve`,
+   * `status`, `resources` and `refusals`. Throws `CrawlNotEnabledError` when the Blueprint has no `crawl` block.
+   *
+   * ```ts
+   * const gov = crawl.create({ instance, name: 'gov', queue, userAgent: 'VerityBot/1.0 (+https://verity.example/bot)',
+   *   accept: ({ contentType }) => contentType === 'application/pdf' })
+   * const { id } = await gov.start({ seed: 'https://www.example.gov.uk/', scope: { domain: 'example.gov.uk' } })
+   * ```
+   *
+   * Every request goes through `untrustedFetch.get`: http(s) only, public addresses only (checked at connect), a byte
+   * limit and a time limit per request, one trace step `pipework/fetch:get` each. robots.txt is read once per host before any
+   * other request to it and obeyed (RFC 9309: the group for the User-Agent's product token, longest match, `Crawl-delay`;
+   * 4xx allows everything; 5xx or no answer disallows, retried three times, then the crawl stops with `stoppedBy: 'robots_unreachable'`).
+   * Requests to a host are `perHostConcurrency` (default 1) at a time and at least the larger of `minGapMs` (default 1000) and
+   * `Crawl-delay` apart. A redirect is not followed inside a request: its target is queued like a link, so it meets scope,
+   * robots and the gap; a target outside the scope is refused as `scope`.
+   *
+   * `accept({ url, contentType })` says which resources are listed; in-scope html is followed for links (read by a scanner that
+   * runs nothing); anything that is neither html nor accepted is refused before its body is read. A crawl that reaches a limit
+   * (`maxPages`, `maxDepth`, `maxBytes`, `maxWallMs`) ends `stopped` with `stoppedBy` naming it.
+   *
+   * State is in the database. The job claims queued URLs one at a time and records each URL's outcome, its links and its
+   * listed resource in one transaction, so a job that dies is retried by the queue and carries on from the queued rows. A URL
+   * finished before the death is never fetched again; the one in flight at the death may be. A later crawl of the same seed
+   * sends `If-None-Match` / `If-Modified-Since` for the resources the earlier one listed and records a 304 as `unchanged`.
+   *
+   * With `crawl: { requireApproval: true }` in the Blueprint, `start` records the crawl as `awaiting_approval` and queues nothing
+   * until `approve(id)`. `start` and `approve` each write one `Crawl` audit record.
+   */
+  create: createCrawl,
+} as const

package/src/infra/crawl/robots.ts

@@ -0,0 +1,100 @@
+// robots.txt per RFC 9309: groups by user agent, longest-match Allow/Disallow with `*` and `$`, Allow wins a tie,
+// Crawl-delay from the matched group. Pure: the text in, a decision function out.
+
+/** What a robots.txt says to one crawler. */
+export interface RobotsRules {
+  /** Whether the crawler may fetch this path and query (`/a/b?x=1`). */
+  allows(pathAndQuery: string): boolean
+  /** Seconds the matched group asks between requests; null when it names none. */
+  readonly crawlDelaySeconds: number | null
+}
+
+interface Group { agents: string[]; rules: Array<{ allow: boolean; pattern: string }>; delay: number | null }
+
+/** Every path is allowed (what a 4xx answer means). */
+export const ALLOW_ALL: RobotsRules = { allows: () => true, crawlDelaySeconds: null }
+/** No path is allowed (what an unreachable robots.txt means). */
+export const DISALLOW_ALL: RobotsRules = { allows: () => false, crawlDelaySeconds: null }
+
+/** The product token of a User-Agent: its leading letters, `-` and `_` (`VerityBot/1.2 (+url)` is `VerityBot`), lower-case. */
+export function productToken(userAgent: string): string {
+  let end = 0
+  while (end < userAgent.length && /[A-Za-z_-]/.test(userAgent[end]!)) end++
+  return userAgent.slice(0, end).toLowerCase()
+}
+
+function parseGroups(text: string): Group[] {
+  const groups: Group[] = []
+  let current: Group | null = null
+  let inAgents = false
+  const lines = text.replace(/^/, '').split(/\r\n|\r|\n/)
+  for (const raw of lines) {
+    const hash = raw.indexOf('#')
+    const line = (hash === -1 ? raw : raw.slice(0, hash)).trim()
+    const colon = line.indexOf(':')
+    if (colon === -1) continue
+    const name = line.slice(0, colon).trim().toLowerCase()
+    const value = line.slice(colon + 1).trim()
+    if (name === 'user-agent') {
+      if (current === null || !inAgents) {
+        current = { agents: [], rules: [], delay: null }
+        groups.push(current)
+      }
+      current.agents.push(value.toLowerCase())
+      inAgents = true
+      continue
+    }
+    inAgents = false
+    if (current === null) continue
+    if (name === 'allow' || name === 'disallow') {
+      if (value !== '') current.rules.push({ allow: name === 'allow', pattern: value })
+    } else if (name === 'crawl-delay') {
+      const n = Number(value)
+      if (Number.isFinite(n) && n >= 0 && current.delay === null) current.delay = n
+    }
+  }
+  return groups
+}
+
+/** Whether `pattern` (with `*` wildcards and an optional trailing `$`) matches the start of `path`; linear in the sizes. */
+function patternMatches(pattern: string, path: string): boolean {
+  const anchored = pattern.endsWith('$')
+  const body = anchored ? pattern.slice(0, -1) : pattern
+  const parts = body.split('*')
+  const first = parts[0]!
+  if (!path.startsWith(first)) return false
+  let pos = first.length
+  if (parts.length === 1) return anchored ? pos === path.length : true
+  for (let i = 1; i < parts.length - 1; i++) {
+    const found = path.indexOf(parts[i]!, pos)
+    if (found === -1) return false
+    pos = found + parts[i]!.length
+  }
+  const last = parts[parts.length - 1]!
+  if (!anchored) return path.indexOf(last, pos) !== -1
+  return path.length - last.length >= pos && path.endsWith(last)
+}
+
+/** Reads robots.txt `text` for the crawler named by `userAgent`: the groups naming its product token, else the `*` groups. */
+export function parseRobots(text: string, userAgent: string): RobotsRules {
+  const groups = parseGroups(text)
+  const token = productToken(userAgent)
+  let matched = groups.filter(g => token !== '' && g.agents.includes(token))
+  if (matched.length === 0) matched = groups.filter(g => g.agents.includes('*'))
+  const rules = matched.flatMap(g => g.rules)
+  const delay = matched.map(g => g.delay).find(d => d !== null) ?? null
+  return {
+    crawlDelaySeconds: delay,
+    allows(pathAndQuery: string): boolean {
+      let best: { allow: boolean; length: number } | null = null
+      for (const rule of rules) {
+        if (!patternMatches(rule.pattern, pathAndQuery)) continue
+        const length = rule.pattern.length
+        if (best === null || length > best.length || (length === best.length && rule.allow && !best.allow)) {
+          best = { allow: rule.allow, length }
+        }
+      }
+      return best === null ? true : best.allow
+    },
+  }
+}

package/src/infra/crawl/schema.ts

@@ -0,0 +1,89 @@
+// The tables behind the site crawler: one row per crawl, one per URL the crawl has met (the frontier and its outcome),
+// one per host (robots.txt and the time of the last request), one per listed resource. Not tenant-scoped; a crawl row
+// records the tenant it was started for.
+
+import { define } from '../../data/domain/index.js'
+import { field } from '../../data/domain/field.js'
+import { schema } from '../../data/schema/namespace.js'
+import { index, uniqueIndex } from '../../data/db/idx/index.js'
+
+export const CRAWL_STATES = ['awaiting_approval', 'running', 'done', 'stopped', 'failed'] as const
+export const CRAWL_URL_STATES = ['queued', 'fetching', 'done', 'refused'] as const
+export const CRAWL_STOPPED_BY = ['pages', 'depth', 'bytes', 'wall', 'robots_unreachable'] as const
+
+export const crawls = define('pipework_crawl', {
+  id: field.uuid().defaultRandom().primaryKey(),
+  site: field.text(),
+  seed: field.text(),
+  scope: field.jsonb(schema.check.json()),
+  limits: field.jsonb(schema.check.json()),
+  state: field.enum(CRAWL_STATES),
+  stoppedBy: field.text().nullable(),
+  error: field.text().nullable(),
+  tenantId: field.text().nullable(),
+  startedBy: field.text().nullable(),
+  approvedBy: field.text().nullable(),
+  bytes: field.bigint().default(0),
+  createdAt: field.timestamp().default('now'),
+  runStartedAt: field.timestamp().nullable(),
+  finishedAt: field.timestamp().nullable(),
+}, {
+  indexes: (t) => ({
+    bySeed: index('idx_pipework_crawl_seed').on(t['site'], t['seed'], t['createdAt']),
+  }),
+})
+
+export const crawlUrls = define('pipework_crawl_url', {
+  id: field.uuid().defaultRandom().primaryKey(),
+  crawlId: field.uuid(),
+  url: field.text(),
+  seq: field.serial(),
+  depth: field.integer(),
+  fromUrl: field.text().nullable(),
+  redirects: field.integer().default(0),
+  state: field.enum(CRAWL_URL_STATES).default('queued'),
+  requested: field.boolean().default(false),
+  seedChain: field.boolean().default(false),
+  reason: field.text().nullable(),
+  message: field.text().nullable(),
+  status: field.integer().nullable(),
+}, {
+  indexes: (t) => ({
+    byUrl: uniqueIndex('uq_pipework_crawl_url').on(t['crawlId'], t['url']),
+    next: index('idx_pipework_crawl_url_next').on(t['crawlId'], t['state'], t['depth'], t['seq']),
+  }),
+})
+
+export const crawlHosts = define('pipework_crawl_host', {
+  id: field.uuid().defaultRandom().primaryKey(),
+  crawlId: field.uuid(),
+  host: field.text(),
+  robotsState: field.text().nullable(),
+  robotsText: field.text().nullable(),
+  robotsAttempts: field.integer().default(0),
+  lastRequestAtMs: field.bigint().default(0),
+}, {
+  indexes: (t) => ({
+    byHost: uniqueIndex('uq_pipework_crawl_host').on(t['crawlId'], t['host']),
+  }),
+})
+
+export const crawlResources = define('pipework_crawl_resource', {
+  id: field.uuid().defaultRandom().primaryKey(),
+  crawlId: field.uuid(),
+  url: field.text(),
+  fromUrl: field.text().nullable(),
+  status: field.integer(),
+  contentType: field.text().nullable(),
+  bytes: field.bigint(),
+  etag: field.text().nullable(),
+  lastModified: field.text().nullable(),
+  sha256: field.text().nullable(),
+  fileId: field.text().nullable(),
+  unchanged: field.boolean().default(false),
+  createdAt: field.timestamp().default('now'),
+}, {
+  indexes: (t) => ({
+    byUrl: uniqueIndex('uq_pipework_crawl_resource').on(t['crawlId'], t['url']),
+  }),
+})