pipework 0.252.0 → 0.253.0

package/src/infra/email/inbound.ts

@@ -0,0 +1,622 @@
+import { createHash, randomUUID } from 'node:crypto'
+import { copyFile, mkdir, readdir, readFile, stat, unlink } from 'node:fs/promises'
+import { basename, join } from 'node:path'
+import { Readable } from 'node:stream'
+import { sql } from '../../data/db/sql.js'
+import { pipeSystem } from '../../data/db/db.js'
+import { schema } from '../../data/schema/namespace.js'
+import { fitting } from '../../request/di/index.js'
+import { getContext } from '../../request/context/store.js'
+import { getBaseLogger } from '../../core/logging/index.js'
+import { recordFault, faultMessage } from '../alerts/faults.js'
+import type { Queue } from '../../async/jobs/queue.js'
+import type { Handler } from '../../request/di/types.js'
+import { __flowStep } from '../../trace/runtime/flow-step.js'
+import type { StorageBackend } from '../storage/backend.js'
+import type { Storage, StoredFile } from '../storage/handle.js'
+import { auditEmail } from './audit.js'
+import { extensionOf, resolveMediaType, SNIFF_BYTES } from '../../request/http/sniff.js'
+import { parseMime, type ParsedAddress } from './mime.js'
+import { snsHandler } from './sns.js'
+import { describeError } from '../../core/logging/error-chain.js'
+
+export const RECEIVE_JOB = 'pipework.email.receive'
+const DEFAULT_RETENTION_DAYS = 30
+const DEFAULT_MAX_RAW_BYTES = 41_943_040
+const DEFAULT_SWEEP_LIMIT = 500
+
+export interface EmailAddress {
+  readonly address: string
+  readonly name: string
+}
+
+export interface ReceivedAttachment {
+  readonly filename: string | null
+  readonly mediaType: string
+  readonly size: number
+  /** Lowercase hex SHA-256 of the decoded content. */
+  readonly sha256: string
+  /** The `Content-ID` without angle brackets, for `cid:` references from the HTML. */
+  readonly contentId: string | null
+  readonly inline: boolean
+  /** The stored file, or `null` when no tenant was resolved or no `storage` was given. */
+  readonly file: StoredFile | null
+}
+
+/** Why a message that had a From field names no author: several From fields, or several mailboxes in one. */
+export type FromProblem = 'several-from-fields' | 'several-mailboxes'
+
+/** What SES concluded about a received message: each of `PASS`, `FAIL`, `GRAY` or `PROCESSING_FAILED`. */
+export interface EmailVerdicts {
+  readonly spam: string | null
+  readonly virus: string | null
+  readonly spf: string | null
+  readonly dkim: string | null
+  readonly dmarc: string | null
+}
+
+/** What is known about a message before it is threaded: everything but its conversation and tenant. Handed to `resolveTenant`. */
+export type UnthreadedEmail = Omit<ReceivedEmail, 'conversationId' | 'isNewConversation' | 'tenantId'>
+
+/** A received, parsed and threaded message, as handed to `onMessage`. */
+export interface ReceivedEmail {
+  /** The `pipework_email_inbound` row id; pass it as `inReplyTo` to `mail.send` to answer. */
+  readonly id: string
+  readonly conversationId: string
+  /** True when no earlier received or sent message matched this one's `In-Reply-To`/`References`. */
+  readonly isNewConversation: boolean
+  readonly tenantId: string | null
+  /** The `Message-ID` without angle brackets, or `null`. */
+  readonly messageId: string | null
+  readonly inReplyTo: string | null
+  readonly references: readonly string[]
+  /** An empty address and name when there is no readable From, or when the author is ambiguous (see `fromProblem`). */
+  readonly from: EmailAddress
+  /**
+   * Why `from` is empty for a message that had a From: several From fields, or several mailboxes in one.
+   * RFC 7489 section 6.6.1: verifiers differ in which one they judge, so no author is named. Null otherwise.
+   */
+  readonly fromProblem: FromProblem | null
+  readonly to: readonly EmailAddress[]
+  readonly cc: readonly EmailAddress[]
+  readonly replyTo: readonly EmailAddress[]
+  readonly subject: string
+  readonly date: Date | null
+  readonly text: string | null
+  readonly html: string | null
+  readonly attachments: readonly ReceivedAttachment[]
+  /**
+   * `null` for a message from the folder stand-in. DMARC vouches for `from` only when `fromProblem` is null
+   * and `from.address` is non-empty; a PASS on a message with an ambiguous From says nothing about any one author.
+   */
+  readonly verdicts: EmailVerdicts | null
+  /** The envelope recipients SES accepted the message for; for the stand-in, the To and Cc addresses. */
+  readonly recipients: readonly string[]
+}
+
+/** A received message's row, without bodies. */
+export interface InboundRecord {
+  readonly id: string
+  readonly source: 'ses' | 'folder'
+  readonly status: 'received' | 'processed' | 'failed'
+  readonly error: string | null
+  readonly messageId: string | null
+  readonly conversationId: string | null
+  readonly tenantId: string | null
+  readonly subject: string | null
+  readonly fromAddress: string | null
+  readonly verdicts: EmailVerdicts | null
+  readonly receivedAt: Date
+  readonly processedAt: Date | null
+  readonly rawExpiresAt: Date
+  readonly rawDeletedAt: Date | null
+}
+
+export interface InboundSweepResult {
+  /** Raw copies deleted. */
+  readonly deleted: number
+  /** Raw copies that could not be deleted (left for the next sweep). */
+  readonly failed: number
+}
+
+export interface InboundScanResult {
+  /** Files taken in. */
+  readonly received: number
+  /** Files already taken in earlier (same bytes); moved to `received/` without a new row. */
+  readonly duplicates: number
+}
+
+type RawLocation = Record<string, unknown>
+
+/** Where received mail comes from. Made by `mail.sesReceipt` or `mail.inboundFolder`. */
+export interface InboundSource {
+  readonly kind: 'ses' | 'folder'
+  /** Reads the raw message; throws `RawTooLargeError` past `maxBytes` and `RawNotFoundError` when it definitely does not exist. */
+  read(location: RawLocation, maxBytes: number): Promise<Buffer>
+  remove(location: RawLocation): Promise<void>
+  readonly ses?: SesReceiptConfig
+  readonly folder?: { readonly dir: string }
+}
+
+export class RawNotFoundError extends Error {
+  constructor(detail: string) {
+    super(`raw message not found: ${detail}`)
+  }
+}
+
+export class RawTooLargeError extends Error {
+  constructor(readonly maxBytes: number) {
+    super(`raw message is larger than ${maxBytes} bytes`)
+  }
+}
+
+export interface SesReceiptConfig {
+  /** A storage backend over the SES receipt bucket with no prefix (`storage.s3({ bucket, region, credentials })`); keys are the S3 action's object keys. */
+  readonly bucket: StorageBackend
+  /** The SNS topics SES publishes receipt notifications to. */
+  readonly topicArns: readonly string[]
+  readonly region?: string
+  readonly fetchCert?: (url: string) => Promise<string>
+  readonly confirm?: (url: string) => Promise<void>
+}
+
+async function collect(stream: AsyncIterable<Buffer | Uint8Array>, maxBytes: number): Promise<Buffer> {
+  const parts: Buffer[] = []
+  let size = 0
+  for await (const chunk of stream) {
+    const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
+    size += buf.length
+    if (size > maxBytes) {
+      if (stream instanceof Readable) stream.destroy()
+      throw new RawTooLargeError(maxBytes)
+    }
+    parts.push(buf)
+  }
+  return Buffer.concat(parts)
+}
+
+/**
+ * The SES receipt source. SES stores each raw message in S3 (receipt rule action "S3" with an SNS topic)
+ * and notifies the topic; `inbound.webhook` verifies the notification and records it, and the receive job
+ * reads the object from `bucket`. A notification whose action is not S3 (for example the SNS action,
+ * which carries the message inline), or whose type is not `Received`, is answered 200 and ignored.
+ */
+export function sesReceipt(config: SesReceiptConfig): InboundSource {
+  if (config.topicArns.length === 0) throw new Error('[pipework] mail.sesReceipt needs at least one topic ARN in topicArns.')
+  return {
+    kind: 'ses',
+    ses: config,
+    async read(location, maxBytes) {
+      let stream: Awaited<ReturnType<StorageBackend['read']>>
+      try {
+        stream = await config.bucket.read(String(location['key']))
+      } catch (err) {
+        if (/\bNoSuchKey\b/.test(describeError(err).message)) throw new RawNotFoundError(String(location['key']))
+        throw err
+      }
+      return collect(stream, maxBytes)
+    },
+    async remove(location) {
+      await config.bucket.remove(String(location['key']))
+    },
+  }
+}
+
+/**
+ * The development source: `.eml` files dropped in `dir` are received by `inbound.scan()`, which copies each
+ * to `<dir>/received/` (where the raw copy stays until `sweep`) and removes the original. Refused when the
+ * environment is `production`.
+ */
+export function inboundFolder(config: { readonly dir: string }): InboundSource {
+  return {
+    kind: 'folder',
+    folder: config,
+    async read(location, maxBytes) {
+      const path = String(location['path'])
+      try {
+        if ((await stat(path)).size > maxBytes) throw new RawTooLargeError(maxBytes)
+        return await readFile(path)
+      } catch (err) {
+        if ((err as { code?: string }).code === 'ENOENT') throw new RawNotFoundError(path)
+        throw err
+      }
+    },
+    async remove(location) {
+      try { await unlink(String(location['path'])) } catch (err) { if ((err as { code?: string }).code !== 'ENOENT') throw err }
+    },
+  }
+}
+
+export interface InboundConfig {
+  readonly source: InboundSource
+  /** The app's job queue; its worker surface must run `inbound.jobs`. */
+  readonly queue: Queue
+  /**
+   * Called in the receive job with each parsed, threaded message. A throw makes the queue retry the job.
+   * It is called at least once per message: a retry after a handler error, or a job re-run after a crash,
+   * calls it again with the same `id`. Make it idempotent on `id`.
+   */
+  readonly onMessage: (email: ReceivedEmail) => Promise<void> | void
+  /** Stores attachments (as the resolved tenant's files). Without it, or without a tenant, attachments have `file: null`. */
+  readonly storage?: Storage
+  /**
+   * Names the tenant a message belongs to (`null` for none), e.g. by its recipient address. It runs before
+   * threading, so it gets the message without a conversation: a message is only joined to a conversation of
+   * the same tenant (or of a message with no tenant).
+   */
+  readonly resolveTenant?: (email: UnthreadedEmail) => Promise<string | null>
+  /** Days the raw copy is kept before `sweep` deletes it. Default 30. */
+  readonly rawRetentionDays?: number
+  /** Days each stored attachment is kept, from receipt, before the storage sweep deletes it. Unset: no expiry. */
+  readonly attachmentRetentionDays?: number
+  /** Raw messages larger than this are marked `failed` and not handled. Default 40 MiB. */
+  readonly maxRawBytes?: number
+  /** The app's environment; `inboundFolder` is refused when this is `production`. */
+  readonly environment?: string
+}
+
+export interface Inbound {
+  /** The public POST endpoint SNS delivers SES receipt notifications to (SES source only). */
+  webhook(config: { readonly path: string }): Handler<Record<string, unknown>, unknown>
+  /** Job handlers for the receive job; spread into the worker surface's `jobs`. */
+  readonly jobs: readonly Handler<Record<string, unknown>, unknown>[]
+  /** Folder source only: receives every `.eml` file in the folder. Needs a Flow (a job or schedule run). */
+  scan(): Promise<InboundScanResult>
+  /** Deletes raw copies past their retention. Needs a Flow (a job or schedule run). */
+  sweep(options?: { readonly limit?: number }): Promise<InboundSweepResult>
+  /** The row of a received message, or `null`. Needs a Flow (a job, request or test context). */
+  get(id: string): Promise<InboundRecord | null>
+}
+
+function toDate(value: unknown): Date {
+  if (value instanceof Date) return value
+  const text = String(value)
+  return new Date(/(Z|[+-]\d\d(:?\d\d)?)$/.test(text) ? text : `${text.replace(' ', 'T')}Z`)
+}
+
+/** A Postgres text-array literal, sent as one parameter and cast with `::text[]`. */
+function textArray(values: readonly string[]): string {
+  return `{${values.map(v => `"${v.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`).join(',')}}`
+}
+
+function verdictsOf(receipt: Record<string, unknown>): EmailVerdicts {
+  const status = (key: string): string | null => {
+    const v = receipt[key] as { status?: unknown } | undefined
+    return typeof v?.status === 'string' ? v.status : null
+  }
+  return {
+    spam: status('spamVerdict'), virus: status('virusVerdict'), spf: status('spfVerdict'),
+    dkim: status('dkimVerdict'), dmarc: status('dmarcVerdict'),
+  }
+}
+
+function addressOf(a: ParsedAddress | null): EmailAddress {
+  return a === null ? { address: '', name: '' } : { address: a.address, name: a.name }
+}
+
+/**
+ * Receiving email. `inbound.webhook` (SES) or `inbound.scan` (folder) records each message in
+ * `pipework_email_inbound` and enqueues the `pipework.email.receive` job, once per message. The job
+ * reads the raw message, parses it, assigns a conversation, resolves the tenant, stores the attachments
+ * as that tenant's files and calls `onMessage` with a `ReceivedEmail`.
+ *
+ * Needs blueprint `email: {}` (table `pipework_email_inbound`; `pipework_email_messages` gains `conversation_id`).
+ * The row is not tenant-scoped (it exists before a tenant is known) and holds no bodies: the raw message
+ * is the copy and `sweep` deletes it after `rawRetentionDays`.
+ *
+ * Threading: the ids in In-Reply-To then References are matched to an earlier received message's
+ * Message-ID, or, for an id `<X@region.amazonses.com>`, to a sent message whose `provider_message_id` is `X`
+ * (set the sent message's conversation with `mail.send({ conversationId })` or `inReplyTo`). The first match
+ * decides the conversation; with none the message starts one (`isNewConversation: true`). Ids are matched exactly.
+ * A received message is matched only when its tenant equals the tenant `resolveTenant` named for this message or
+ * either is null; a sent message's conversation is refused when it holds a received message of another tenant.
+ *
+ * Threading is not authentication: Message-IDs are not secret and From is forgeable. Before attaching a message to
+ * a conversation's participants, the app checks `from` against them and the DKIM and DMARC verdicts.
+ *
+ * The receive job is a trace step `pipework/email:receive` with an `EmailInbound:received` audit record on success.
+ * A handler error (or a failure reading the raw message) rethrows so the queue retries; attachments stored by
+ * an earlier attempt are reused. Set `maxAttempts` on the queue. A raw message over `maxRawBytes` is marked
+ * `failed` and not retried. Message bodies are never logged.
+ */
+export function createInbound(config: InboundConfig): Inbound {
+  if (config.source.kind === 'folder' && config.environment === 'production') {
+    throw new Error(
+      '[pipework] mail.inboundFolder is refused in production.\n\n' +
+      '  The stand-in reads .eml files from a folder.\n' +
+      '  Configure a real source, e.g. mail.sesReceipt({ bucket, topicArns }).\n',
+    )
+  }
+  const retentionDays = Math.floor(config.rawRetentionDays ?? DEFAULT_RETENTION_DAYS)
+  const maxRawBytes = config.maxRawBytes ?? DEFAULT_MAX_RAW_BYTES
+  if (!(retentionDays >= 0)) throw new Error('[pipework] inbound rawRetentionDays must be zero or more.')
+  const attachmentDays = config.attachmentRetentionDays === undefined ? undefined : Math.floor(config.attachmentRetentionDays)
+  if (attachmentDays !== undefined && !(attachmentDays >= 1)) throw new Error('[pipework] inbound attachmentRetentionDays must be one or more.')
+  if (!(maxRawBytes > 0)) throw new Error('[pipework] inbound maxRawBytes must be positive.')
+
+  async function record(
+    source: 'ses' | 'folder', sourceId: string, location: RawLocation,
+    extra: { verdicts: EmailVerdicts | null; recipients: readonly string[] | null; from: string | null },
+  ): Promise<string | null> {
+    const traceId = getContext()?.trace?.id ?? null
+    return pipeSystem().transaction(async (tx) => {
+      const inserted = await tx.execute(sql`
+        INSERT INTO pipework_email_inbound
+          (source_id, source, raw_location, raw_expires_at, verdicts, recipients, from_address, trace_id)
+        VALUES (${sourceId}, ${source}, ${JSON.stringify(location)}::jsonb, now() + make_interval(days => ${retentionDays}::int),
+          ${extra.verdicts === null ? null : JSON.stringify(extra.verdicts)}::jsonb,
+          ${extra.recipients === null ? null : JSON.stringify(extra.recipients)}::jsonb, ${extra.from}, ${traceId}::uuid)
+        ON CONFLICT (source_id) DO NOTHING
+        RETURNING id`)
+      if (inserted.length === 0) return null
+      const inboundId = inserted[0]!['id'] as string
+      await config.queue.enqueue(tx, { type: RECEIVE_JOB, payload: { inboundId }, continueFlow: true })
+      return inboundId
+    })
+  }
+
+  async function ingestNotification(message: string): Promise<Record<string, unknown>> {
+    let parsed: unknown
+    try {
+      parsed = JSON.parse(message)
+    } catch {
+      return { ignored: true }
+    }
+    if (typeof parsed !== 'object' || parsed === null) return { ignored: true }
+    const event = parsed as Record<string, unknown>
+    if (event['notificationType'] !== 'Received') return { ignored: true }
+    const mailPart = event['mail'] as { messageId?: unknown; source?: unknown; destination?: unknown } | undefined
+    const receipt = event['receipt'] as Record<string, unknown> | undefined
+    const action = receipt?.['action'] as { type?: unknown; bucketName?: unknown; objectKey?: unknown } | undefined
+    if (typeof mailPart?.messageId !== 'string' || receipt === undefined) return { ignored: true }
+    if (action?.type !== 'S3' || typeof action.bucketName !== 'string' || typeof action.objectKey !== 'string') {
+      getBaseLogger().info({ event: 'email receipt ignored', actionType: action?.type ?? null }, 'SES receipt ignored: its action is not S3')
+      return { ignored: true }
+    }
+    const recipientList = Array.isArray(receipt['recipients']) ? receipt['recipients'] : mailPart.destination
+    const recipients = Array.isArray(recipientList) ? recipientList.filter((r): r is string => typeof r === 'string') : []
+    const id = await record('ses', mailPart.messageId, { bucket: action.bucketName, key: action.objectKey }, {
+      verdicts: verdictsOf(receipt), recipients, from: typeof mailPart.source === 'string' ? mailPart.source : null,
+    })
+    if (id === null) getBaseLogger().info({ event: 'email receipt duplicate', sourceId: mailPart.messageId }, 'SES receipt already recorded')
+    return {}
+  }
+
+  interface Threaded { readonly conversationId: string; readonly isNew: boolean }
+
+  async function thread(ids: readonly string[], recorded: string | null, tenantId: string | null): Promise<Threaded> {
+    const db = pipeSystem()
+    if (ids.length > 0) {
+      const sesLocals = ids.flatMap(id => {
+        const m = /^(.+)@[^@]*amazonses\.com$/i.exec(id)
+        return m === null ? [] : [m[1]!]
+      })
+      const received = await db.execute(sql`
+        SELECT message_id_header AS id, conversation_id FROM pipework_email_inbound
+        WHERE message_id_header = ANY(${textArray(ids)}::text[]) AND conversation_id IS NOT NULL
+          AND (${tenantId}::uuid IS NULL OR tenant_id IS NULL OR tenant_id = ${tenantId}::uuid)`)
+      const sent = sesLocals.length === 0 ? [] : await db.execute(sql`
+        SELECT provider_message_id AS id, conversation_id FROM pipework_email_messages
+        WHERE provider_message_id = ANY(${textArray(sesLocals)}::text[]) AND conversation_id IS NOT NULL`)
+      const byId = new Map<string, string>()
+      for (const r of received) byId.set(r['id'] as string, r['conversation_id'] as string)
+      const bySes = new Map<string, string>()
+      for (const r of sent) {
+        const conversation = r['conversation_id'] as string
+        const foreign = tenantId === null ? [] : await db.execute(sql`
+          SELECT 1 FROM pipework_email_inbound
+          WHERE conversation_id = ${conversation}::uuid AND tenant_id IS NOT NULL AND tenant_id <> ${tenantId}::uuid LIMIT 1`)
+        if (foreign.length === 0) bySes.set(r['id'] as string, conversation)
+      }
+      for (const id of ids) {
+        const hit = byId.get(id) ?? bySes.get(/^(.+)@[^@]*amazonses\.com$/i.exec(id)?.[1] ?? '\u0000')
+        if (hit !== undefined) return { conversationId: hit, isNew: false }
+      }
+    }
+    // No match: this message's own conversation, kept from an earlier attempt of the job.
+    if (recorded !== null) return { conversationId: recorded, isNew: true }
+    return { conversationId: randomUUID(), isNew: true }
+  }
+
+  async function fail(id: string, error: string): Promise<{ status: string }> {
+    await pipeSystem().execute(sql`UPDATE pipework_email_inbound SET status = 'failed', failed_at = now(), error = ${error} WHERE id = ${id}::uuid`)
+    await auditEmail('failed', id, { error }, 'EmailInbound')
+    getBaseLogger().error({ event: 'email receive failed', inboundId: id }, `received email not handled: ${error}`)
+    return { status: 'failed' }
+  }
+
+  const receive = __flowStep({ ref: 'pipework/email:receive' }, async (inboundId: string): Promise<{ status: string }> => {
+    const db = pipeSystem()
+    const row = (await db.execute(sql`SELECT * FROM pipework_email_inbound WHERE id = ${inboundId}::uuid`))[0]
+    if (row === undefined) throw new Error(`[pipework] Received email ${inboundId} does not exist.`)
+    const status = row['status'] as string
+    if (status === 'processed' || status === 'failed') return { status }
+    if (config.source.kind === 'folder' && getContext()?.instance.config.environment === 'production') {
+      throw new Error('[pipework] mail.inboundFolder is refused in production.')
+    }
+    if (row['raw_deleted_at'] !== null) return fail(inboundId, 'the raw message was deleted before it was handled')
+
+    let raw: Buffer
+    try {
+      raw = await config.source.read(row['raw_location'] as RawLocation, maxRawBytes)
+    } catch (err) {
+      if (err instanceof RawTooLargeError || err instanceof RawNotFoundError) return fail(inboundId, err.message)
+      const message = describeError(err).message
+      await db.execute(sql`UPDATE pipework_email_inbound SET error = ${message} WHERE id = ${inboundId}::uuid`)
+      throw err
+    }
+
+    const parsed = parseMime(raw)
+    const recipients = (row['recipients'] as string[] | null) ?? [...parsed.to, ...parsed.cc].map(a => a.address)
+    const verdicts = (row['verdicts'] as EmailVerdicts | null) ?? null
+    // A declared type the bytes contradict resolves to '': store octet-stream rather than refuse the mail.
+    const resolved = parsed.attachments.map(a => {
+      const declared = a.mediaType.split(';')[0]!.trim().toLowerCase()
+      const { detectedType, mediaType } = resolveMediaType({ head: a.content.subarray(0, SNIFF_BYTES), declared, extension: extensionOf(a.filename) })
+      return { mediaType: mediaType === '' ? 'application/octet-stream' : mediaType, detectedType, declaredType: declared }
+    })
+    const unthreaded: UnthreadedEmail = {
+      id: inboundId,
+      messageId: parsed.messageId, inReplyTo: parsed.inReplyTo, references: parsed.references,
+      from: addressOf(parsed.from), fromProblem: parsed.fromProblem,
+      to: parsed.to.map(addressOf), cc: parsed.cc.map(addressOf), replyTo: parsed.replyTo.map(addressOf),
+      subject: parsed.subject, date: parsed.date, text: parsed.text, html: parsed.html,
+      attachments: parsed.attachments.map((a, idx) => ({
+        filename: a.filename, mediaType: resolved[idx]!.mediaType, size: a.size, sha256: createHash('sha256').update(a.content).digest('hex'),
+        contentId: a.contentId, inline: a.inline, file: null,
+      })),
+      verdicts, recipients,
+    }
+
+    let tenantId = (row['tenant_id'] as string | null) ?? null
+    if (tenantId === null && config.resolveTenant !== undefined) tenantId = await config.resolveTenant(unthreaded)
+    const { conversationId, isNew } = await thread(parsed.threadIds, (row['conversation_id'] as string | null) ?? null, tenantId)
+    const base: ReceivedEmail = { ...unthreaded, conversationId, isNewConversation: isNew, tenantId }
+
+    await db.execute(sql`
+      UPDATE pipework_email_inbound SET
+        message_id_header = ${parsed.messageId}, in_reply_to = ${parsed.inReplyTo}, "references" = ${JSON.stringify(parsed.references)}::jsonb,
+        from_address = ${base.from.address}, subject = ${parsed.subject}, conversation_id = ${conversationId}::uuid,
+        tenant_id = ${tenantId}::uuid, recipients = ${JSON.stringify(recipients)}::jsonb, error = NULL
+      WHERE id = ${inboundId}::uuid`)
+
+    let attachments = base.attachments
+    if (config.storage !== undefined && tenantId !== null && parsed.attachments.length > 0) {
+      const storage = config.storage
+      const stored: (string | null)[] = Array.isArray(row['attachment_files']) ? [...(row['attachment_files'] as (string | null)[])] : []
+      const files: (StoredFile | null)[] = []
+      for (let i = 0; i < parsed.attachments.length; i++) {
+        const part = parsed.attachments[i]!
+        let file: StoredFile | null = null
+        const earlier = stored[i]
+        if (typeof earlier === 'string') file = await storage.getFor(tenantId, earlier)
+        if (file === null) {
+          file = await storage.putFor(tenantId, {
+            body: Readable.from([part.content]), mediaType: attachments[i]!.mediaType,
+            metadata: { detectedType: resolved[i]!.detectedType, declaredType: resolved[i]!.declaredType },
+            ...(part.filename !== null ? { filename: part.filename } : {}),
+            expectedSha256: attachments[i]!.sha256,
+            ...(attachmentDays !== undefined ? { expiresAt: new Date(toDate(row['received_at']).getTime() + attachmentDays * 86_400_000) } : {}),
+          })
+          stored[i] = file.id
+          await db.execute(sql`UPDATE pipework_email_inbound SET attachment_files = ${JSON.stringify(stored)}::jsonb WHERE id = ${inboundId}::uuid`)
+        }
+        files.push(file)
+      }
+      attachments = attachments.map((a, i) => ({ ...a, file: files[i] ?? null }))
+    }
+
+    try {
+      await config.onMessage({ ...base, tenantId, attachments })
+    } catch (err) {
+      const message = describeError(err).message
+      await db.execute(sql`UPDATE pipework_email_inbound SET error = ${message} WHERE id = ${inboundId}::uuid`)
+      throw err
+    }
+    const traceId = getContext()?.trace?.id ?? null
+    await db.execute(sql`
+      UPDATE pipework_email_inbound SET status = 'processed', processed_at = now(), error = NULL, trace_id = COALESCE(${traceId}::uuid, trace_id)
+      WHERE id = ${inboundId}::uuid`)
+    await auditEmail('received', inboundId, {
+      conversationId, isNewConversation: isNew, tenantId, attachments: attachments.length,
+    }, 'EmailInbound')
+    return { status: 'processed' }
+  })
+
+  const receiveJob = fitting.use().job(RECEIVE_JOB)
+    .input(schema.check.object({ inboundId: schema.check.string() }))
+    .fit(async ({ input }: { input: { inboundId: string } }) => receive(input.inboundId))
+
+  return {
+    webhook(webhook) {
+      const ses = config.source.ses
+      if (config.source.kind !== 'ses' || ses === undefined) {
+        throw new Error('[pipework] inbound.webhook needs a mail.sesReceipt source; this inbound reads a folder.')
+      }
+      return snsHandler({
+        path: webhook.path, topicArns: ses.topicArns,
+        ...(ses.region !== undefined ? { region: ses.region } : {}),
+        ...(ses.fetchCert !== undefined ? { fetchCert: ses.fetchCert } : {}),
+        ...(ses.confirm !== undefined ? { confirm: ses.confirm } : {}),
+      }, ingestNotification)
+    },
+    jobs: [receiveJob as unknown as Handler<Record<string, unknown>, unknown>],
+
+    async scan() {
+      const folder = config.source.folder
+      if (config.source.kind !== 'folder' || folder === undefined) {
+        throw new Error('[pipework] inbound.scan needs a mail.inboundFolder source; this inbound receives through SES.')
+      }
+      let names: string[]
+      try {
+        names = (await readdir(folder.dir)).filter(n => n.toLowerCase().endsWith('.eml')).sort()
+      } catch (err) {
+        if ((err as { code?: string }).code === 'ENOENT') return { received: 0, duplicates: 0 }
+        throw err
+      }
+      const done = join(folder.dir, 'received')
+      let received = 0
+      let duplicates = 0
+      for (const name of names) {
+        const from = join(folder.dir, name)
+        if (!(await stat(from)).isFile()) continue
+        const bytes = await readFile(from)
+        const sha = createHash('sha256').update(bytes).digest('hex')
+        await mkdir(done, { recursive: true })
+        const to = join(done, `${sha.slice(0, 12)}-${basename(name)}`)
+        await copyFile(from, to)
+        const id = await record('folder', `folder:${sha}`, { path: to }, { verdicts: null, recipients: null, from: null })
+        await unlink(from)
+        if (id === null) duplicates++
+        else received++
+      }
+      return { received, duplicates }
+    },
+
+    async sweep(options) {
+      const limit = options?.limit ?? DEFAULT_SWEEP_LIMIT
+      const db = pipeSystem()
+      const due = await db.execute(sql`
+        SELECT id, raw_location FROM pipework_email_inbound
+        WHERE raw_expires_at <= now() AND raw_deleted_at IS NULL
+        ORDER BY raw_expires_at LIMIT ${limit}`)
+      let deleted = 0
+      let failed = 0
+      for (const row of due) {
+        try {
+          await config.source.remove(row['raw_location'] as RawLocation)
+          await db.execute(sql`UPDATE pipework_email_inbound SET raw_deleted_at = now() WHERE id = ${row['id'] as string}::uuid`)
+          deleted++
+        } catch (err) {
+          failed++
+          getBaseLogger().error({ err, inboundId: row['id'] }, 'inbound sweep could not delete a raw message')
+          void recordFault('inbound-raw-delete', faultMessage(err), { inboundId: String(row['id']) })
+        }
+      }
+      return { deleted, failed }
+    },
+
+    async get(id) {
+      if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id)) return null
+      const row = (await pipeSystem().execute(sql`SELECT * FROM pipework_email_inbound WHERE id = ${id}::uuid`))[0]
+      if (row === undefined) return null
+      const date = (v: unknown): Date | null => (v === null || v === undefined ? null : toDate(v))
+      return {
+        id: row['id'] as string,
+        source: row['source'] as 'ses' | 'folder',
+        status: row['status'] as 'received' | 'processed' | 'failed',
+        error: (row['error'] as string | null) ?? null,
+        messageId: (row['message_id_header'] as string | null) ?? null,
+        conversationId: (row['conversation_id'] as string | null) ?? null,
+        tenantId: (row['tenant_id'] as string | null) ?? null,
+        subject: (row['subject'] as string | null) ?? null,
+        fromAddress: (row['from_address'] as string | null) ?? null,
+        verdicts: (row['verdicts'] as EmailVerdicts | null) ?? null,
+        receivedAt: toDate(row['received_at']),
+        processedAt: date(row['processed_at']),
+        rawExpiresAt: toDate(row['raw_expires_at']),
+        rawDeletedAt: date(row['raw_deleted_at']),
+      }
+    },
+  }
+}

package/src/infra/email/index.ts

@@ -0,0 +1,11 @@
+export { createMail, normalizeAddress, SEND_JOB, type Mail, type MailConfig, type SendInput, type SendResult,
+  type RenderedEmail, type EmailTemplates, type EmailRenderMeta } from './mail.js'
+export { ses, devStandIn, type MailProvider, type OutgoingMessage, type SesConfig } from './providers.js'
+export {
+  createInbound, sesReceipt, inboundFolder, RECEIVE_JOB, type Inbound, type InboundConfig, type InboundSource, type InboundRecord,
+  type InboundScanResult, type InboundSweepResult, type ReceivedEmail, type ReceivedAttachment, type EmailAddress,
+  type EmailVerdicts, type FromProblem, type UnthreadedEmail, type SesReceiptConfig,
+} from './inbound.js'
+export { snsWebhook, type SnsWebhookConfig } from './sns.js'
+export { signV4, type SigV4Request, type AwsCredentials } from '../aws/sigv4.js'
+export { awsCredentials, type AwsCredentialsOptions } from '../aws/credentials.js'