pipework 0.252.0 → 0.252.1

package/src/infra/email/sns.ts

@@ -0,0 +1,200 @@
+import { createPublicKey, verify as cryptoVerify } from 'node:crypto'
+import { sql } from '../../data/db/sql.js'
+import { pipeSystem } from '../../data/db/db.js'
+import { Builder } from '../../request/di/builder.js'
+import { UnauthorizedError, ValidationError } from '../../core/errors/index.js'
+import { getBaseLogger } from '../../core/logging/index.js'
+import type { Handler } from '../../request/di/types.js'
+import { __flowStep } from '../../trace/runtime/flow-step.js'
+import { auditEmail } from './audit.js'
+import { normalizeAddress } from './mail.js'
+
+export interface SnsWebhookConfig {
+  readonly path: string
+  /** The SNS topics whose messages are accepted. SNS signs every topic, so without this anyone could subscribe the endpoint to their own topic. */
+  readonly topicArns: readonly string[]
+  /** The SNS region; certificate and SubscribeURL hosts must be `sns.<region>.amazonaws.com`. Default: any AWS region. */
+  readonly region?: string
+  /** Fetches the signing certificate PEM. Default: `fetch`. Tests serve their own. */
+  readonly fetchCert?: (url: string) => Promise<string>
+  /** Confirms a subscription by GET of its SubscribeURL. Default: `fetch`. */
+  readonly confirm?: (url: string) => Promise<void>
+}
+
+type SnsMessage = Record<string, string | undefined>
+
+const NOTIFICATION_FIELDS = ['Message', 'MessageId', 'Subject', 'Timestamp', 'TopicArn', 'Type']
+const CONFIRMATION_FIELDS = ['Message', 'MessageId', 'SubscribeURL', 'Timestamp', 'Token', 'TopicArn', 'Type']
+
+export function snsStringToSign(msg: SnsMessage): string {
+  const names = msg['Type'] === 'Notification' ? NOTIFICATION_FIELDS : CONFIRMATION_FIELDS
+  return names.filter(n => msg[n] !== undefined).map(n => `${n}\n${msg[n]!}\n`).join('')
+}
+
+function isSnsHost(url: string, region: string | undefined): boolean {
+  let u: URL
+  try { u = new URL(url) } catch { return false }
+  if (u.protocol !== 'https:' || u.username !== '' || u.password !== '' || u.port !== '') return false
+  const host = region !== undefined ? `sns.${region}.amazonaws.com` : null
+  return host !== null ? u.hostname === host : /^sns\.[a-z0-9-]+\.amazonaws\.com$/.test(u.hostname)
+}
+
+async function defaultFetchCert(url: string): Promise<string> {
+  const res = await fetch(url, { redirect: 'error' })
+  if (!res.ok) throw new Error(`certificate fetch answered HTTP ${res.status}`)
+  return res.text()
+}
+
+async function defaultConfirm(url: string): Promise<void> {
+  const res = await fetch(url, { redirect: 'error' })
+  if (!res.ok) throw new Error(`subscription confirmation answered HTTP ${res.status}`)
+}
+
+function refuse(reason: string): never {
+  getBaseLogger().warn({ event: 'sns message refused', reason }, 'SNS message refused')
+  throw new UnauthorizedError('SNS message verification failed.\n')
+}
+
+interface Recipient { address: string; bounceType: string | null; kind: 'bounce' | 'complaint' }
+
+function recipientsOf(event: Record<string, unknown>): { recipients: Recipient[]; sesMessageId: string | null } {
+  const type = String(event['eventType'] ?? event['notificationType'] ?? '').toLowerCase()
+  const mailPart = event['mail'] as { messageId?: unknown } | undefined
+  const sesMessageId = typeof mailPart?.messageId === 'string' ? mailPart.messageId : null
+  const list = (v: unknown): string[] =>
+    Array.isArray(v) ? (v as ({ emailAddress?: unknown } | null)[]).flatMap(r => (typeof r?.emailAddress === 'string' ? [r.emailAddress] : [])) : []
+  if (type === 'bounce') {
+    const bounce = event['bounce'] as { bounceType?: unknown; bouncedRecipients?: unknown } | undefined
+    const bounceType = typeof bounce?.bounceType === 'string' ? bounce.bounceType : null
+    return { recipients: list(bounce?.bouncedRecipients).map(address => ({ address, bounceType, kind: 'bounce' as const })), sesMessageId }
+  }
+  if (type === 'complaint') {
+    const complaint = event['complaint'] as { complainedRecipients?: unknown } | undefined
+    return { recipients: list(complaint?.complainedRecipients).map(address => ({ address, bounceType: null, kind: 'complaint' as const })), sesMessageId }
+  }
+  return { recipients: [], sesMessageId }
+}
+
+const processEvent = __flowStep({ ref: 'pipework/email:notification' }, async (event: Record<string, unknown>): Promise<void> => {
+  const { recipients, sesMessageId } = recipientsOf(event)
+  const db = pipeSystem()
+  for (const r of recipients) {
+    const address = normalizeAddress(r.address)
+    const found = sesMessageId === null ? [] : await db.execute(sql`
+      SELECT id, trace_id FROM pipework_email_messages WHERE provider_message_id = ${sesMessageId}`)
+    const messageId = (found[0]?.['id'] as string | undefined) ?? null
+    const messageTraceId = (found[0]?.['trace_id'] as string | undefined) ?? null
+    const permanent = r.kind === 'complaint' || r.bounceType === 'Permanent'
+    if (permanent) {
+      await db.execute(sql`
+        INSERT INTO pipework_email_suppressions (address, reason, detail)
+        VALUES (${address}, ${r.kind}, ${JSON.stringify({ bounceType: r.bounceType, sesMessageId })}::jsonb)
+        ON CONFLICT (address) DO UPDATE SET reason = EXCLUDED.reason, detail = EXCLUDED.detail`)
+      getBaseLogger().warn({ event: 'email suppressed', address, reason: r.kind }, 'email suppressed')
+    } else {
+      getBaseLogger().info({ event: 'email bounce not suppressed', address, bounceType: r.bounceType }, 'transient bounce: address kept')
+    }
+    await auditEmail(r.kind, messageId, { address, bounceType: r.bounceType, suppressed: permanent, messageTraceId })
+  }
+})
+
+export interface SnsHandlerConfig {
+  readonly path: string
+  readonly topicArns: readonly string[]
+  readonly region?: string
+  readonly fetchCert?: (url: string) => Promise<string>
+  readonly confirm?: (url: string) => Promise<void>
+}
+
+/**
+ * The shared SNS endpoint: parses the body, verifies the signature (SignatureVersion 1 or 2) against the
+ * certificate at `SigningCertURL`, checks `TopicArn` against `topicArns`, confirms subscriptions, and
+ * passes each verified `Notification`'s `Message` text to `onNotification`. Anything unverified is a 401
+ * before `onNotification` runs. Used by `snsWebhook` and the inbound mail webhook.
+ */
+export function snsHandler(
+  config: SnsHandlerConfig,
+  onNotification: (message: string) => Promise<Record<string, unknown> | void>,
+): Handler<Record<string, unknown>, unknown> {
+  if (config.topicArns.length === 0) throw new Error('[pipework] An SNS webhook needs at least one topic ARN in topicArns.')
+  const fetchCert = config.fetchCert ?? defaultFetchCert
+  const confirm = config.confirm ?? defaultConfirm
+  const certs = new Map<string, string>()
+
+  async function verifySignature(msg: SnsMessage): Promise<void> {
+    const url = msg['SigningCertURL']
+    const signature = msg['Signature']
+    const version = msg['SignatureVersion']
+    if (url === undefined || signature === undefined) refuse('missing signature fields')
+    if (version !== '1' && version !== '2') refuse('unknown SignatureVersion')
+    if (!isSnsHost(url, config.region) || !new URL(url).pathname.endsWith('.pem')) refuse('SigningCertURL is not an SNS certificate URL')
+    let pem = certs.get(url)
+    if (pem === undefined) {
+      try { pem = await fetchCert(url) } catch { refuse('certificate could not be fetched') }
+      certs.set(url, pem)
+    }
+    let ok = false
+    try {
+      ok = cryptoVerify(version === '1' ? 'RSA-SHA1' : 'RSA-SHA256', Buffer.from(snsStringToSign(msg)), createPublicKey(pem), Buffer.from(signature, 'base64'))
+    } catch { ok = false }
+    if (!ok) refuse('signature does not match')
+    if (msg['TopicArn'] === undefined || !config.topicArns.includes(msg['TopicArn'])) refuse('TopicArn is not a configured topic')
+  }
+
+  return new Builder()
+    .public()
+    .rawBody()
+    .request()
+    .route('POST', config.path)
+    .fit(async ({ rawBody }) => {
+      let parsed: unknown
+      try {
+        parsed = JSON.parse(rawBody.toString('utf-8'))
+      } catch {
+        throw new ValidationError({ body: ['is not JSON'] }, 'SNS message is not JSON.')
+      }
+      if (typeof parsed !== 'object' || parsed === null) throw new ValidationError({ body: ['is not an object'] }, 'SNS message is not an object.')
+      const msg = parsed as SnsMessage
+      await verifySignature(msg)
+
+      if (msg['Type'] === 'SubscriptionConfirmation') {
+        const subscribeUrl = msg['SubscribeURL']
+        if (subscribeUrl === undefined || !isSnsHost(subscribeUrl, config.region)) refuse('SubscribeURL is not an SNS URL')
+        await confirm(subscribeUrl)
+        return { status: 200, body: { ok: true, confirmed: true } }
+      }
+      if (msg['Type'] === 'Notification') {
+        const extra = await onNotification(msg['Message'] ?? '')
+        return { status: 200, body: { ok: true, ...(extra ?? {}) } }
+      }
+      return { status: 200, body: { ok: true } }
+    })
+}
+
+/**
+ * Receives SES bounce and complaint events delivered through SNS (HTTP/S subscription).
+ * A permanent bounce or any complaint puts the address (lowercased) on
+ * `pipework_email_suppressions`, which the send job checks; a transient or undetermined
+ * bounce is only logged. Each one is a trace step with an `Email:bounce` / `Email:complaint`
+ * audit record whose `after` carries `messageTraceId`, the trace of the original send.
+ * A suppression is logged at warn as `email suppressed`; an app raises an alert for it with `alerts.create(...).raise`.
+ *
+ * Only messages from the configured `topicArns` are accepted (401 otherwise, before any subscription is
+ * confirmed or event processed): SNS signs every topic, including an attacker's own.
+ * Every message's signature (SignatureVersion 1 or 2) is checked against the certificate
+ * at `SigningCertURL`, which must be `https://sns.<region>.amazonaws.com/...`; a failure is a 401.
+ * A SubscriptionConfirmation is confirmed by fetching its `SubscribeURL`.
+ * SES events delivered through EventBridge are not supported.
+ */
+export function snsWebhook(config: SnsWebhookConfig): Handler<Record<string, unknown>, unknown> {
+  if (config.topicArns.length === 0) throw new Error('[pipework] mail.snsWebhook needs at least one topic ARN in topicArns.')
+  return snsHandler(config, async (message) => {
+    let event: Record<string, unknown>
+    try {
+      event = JSON.parse(message) as Record<string, unknown>
+    } catch {
+      return { ignored: true }
+    }
+    await processEvent(event)
+  })
+}

package/src/infra/email/unsubscribe.ts

@@ -0,0 +1,44 @@
+import { createHmac, timingSafeEqual } from 'node:crypto'
+
+/** A list name: what `SendInput.list` accepts. */
+export const LIST_RE = /^[a-z0-9][a-z0-9._-]{0,63}$/i
+export const MIN_UNSUBSCRIBE_SECRET = 32
+
+const DOMAIN = 'pipework mail unsubscribe\0'
+
+const b64 = (data: Buffer | string): string => Buffer.from(data).toString('base64url')
+
+function signature(secret: string, payload: string): Buffer {
+  return createHmac('sha256', secret).update(DOMAIN + payload).digest()
+}
+
+/** The signed token naming a normalized address and a list; it never expires. */
+export function signUnsubscribeToken(secret: string, address: string, list: string): string {
+  const payload = b64(JSON.stringify({ a: address, l: list }))
+  return `${payload}.${b64(signature(secret, payload))}`
+}
+
+/** The address and list a token names, or null for anything malformed, tampered or signed with another secret. Never throws. */
+export function verifyUnsubscribeToken(secret: string, token: unknown): { readonly address: string; readonly list: string } | null {
+  if (typeof token !== 'string') return null
+  const parts = token.split('.')
+  if (parts.length !== 2) return null
+  const [payload, given] = parts as [string, string]
+  const expected = signature(secret, payload)
+  const got = Buffer.from(given, 'base64url')
+  if (got.length !== expected.length || !timingSafeEqual(got, expected)) return null
+  try {
+    const parsed = JSON.parse(Buffer.from(payload, 'base64url').toString('utf8')) as { a?: unknown; l?: unknown }
+    if (typeof parsed.a !== 'string' || typeof parsed.l !== 'string' || parsed.a === '' || !LIST_RE.test(parsed.l)) return null
+    return { address: parsed.a, list: parsed.l }
+  } catch {
+    return null
+  }
+}
+
+/** The unsubscribe link for a token: `url` with `t` set. */
+export function unsubscribeLink(url: string, token: string): string {
+  const u = new URL(url)
+  u.searchParams.set('t', token)
+  return u.toString()
+}

package/src/infra/fetch/address.ts

@@ -0,0 +1,88 @@
+import { isIP } from 'node:net'
+
+type Cidr4 = readonly [number, number]
+
+/** IPv4 ranges a stranger's URL may not reach: [first address as uint32, prefix length]. */
+const REFUSED_V4: readonly Cidr4[] = [
+  [0x00000000, 8], [0x0a000000, 8], [0x64400000, 10], [0x7f000000, 8], [0xa9fe0000, 16],
+  [0xac100000, 12], [0xc0000000, 24], [0xc0a80000, 16], [0xc6120000, 15], [0xe0000000, 4], [0xf0000000, 4],
+]
+
+function parseV4(text: string): number[] | null {
+  const parts = text.split('.')
+  if (parts.length !== 4) return null
+  const out: number[] = []
+  for (const p of parts) {
+    if (!/^\d{1,3}$/.test(p)) return null
+    const n = Number(p)
+    if (n > 255) return null
+    out.push(n)
+  }
+  return out
+}
+
+/** The 16 bytes of an IPv6 address (dotted tail and zone id allowed), or null when it is not one. */
+function parseV6(input: string): number[] | null {
+  let text = input
+  const zone = text.indexOf('%')
+  if (zone !== -1) text = text.slice(0, zone)
+  let tail: number[] = []
+  const lastColon = text.lastIndexOf(':')
+  if (text.includes('.')) {
+    const v4 = parseV4(text.slice(lastColon + 1))
+    if (v4 === null) return null
+    tail = v4
+    text = text.slice(0, lastColon + 1) + '0:0'
+  }
+  const halves = text.split('::')
+  if (halves.length > 2) return null
+  const groups = (s: string): string[] => (s === '' ? [] : s.split(':'))
+  const head = groups(halves[0]!)
+  const rest = halves.length === 2 ? groups(halves[1]!) : []
+  const missing = 8 - head.length - rest.length
+  if (halves.length === 2 ? missing < 1 : missing !== 0) return null
+  const all = [...head, ...(halves.length === 2 ? new Array<string>(missing).fill('0') : []), ...rest]
+  const bytes: number[] = []
+  for (const g of all) {
+    if (!/^[0-9a-f]{1,4}$/i.test(g)) return null
+    const n = parseInt(g, 16)
+    bytes.push(n >> 8, n & 0xff)
+  }
+  if (tail.length === 4) bytes.splice(12, 4, ...tail)
+  return bytes
+}
+
+function v4Refused(b: readonly number[]): boolean {
+  const n = ((b[0]! << 24) | (b[1]! << 16) | (b[2]! << 8) | b[3]!) >>> 0
+  return REFUSED_V4.some(([base, len]) => {
+    const mask = len === 0 ? 0 : (0xffffffff << (32 - len)) >>> 0
+    return ((n & mask) >>> 0) === base
+  })
+}
+
+/**
+ * True when `address` (IPv4 or IPv6 text, no brackets) is a public address the fetch may connect to.
+ * Anything that is not an IP address, and every private, loopback, link-local, CGNAT, unique-local, site-local,
+ * multicast, reserved, unspecified, documentation or IPv4-embedded form of those, is not public.
+ */
+export function isPublicAddress(address: string): boolean {
+  const kind = isIP(address)
+  if (kind === 4) {
+    const b = parseV4(address)
+    return b !== null && !v4Refused(b)
+  }
+  if (kind !== 6) return false
+  const b = parseV6(address)
+  if (b === null) return false
+  const first96Zero = b.slice(0, 12).every(x => x === 0)
+  if (first96Zero) return false // ::, ::1 and the deprecated IPv4-compatible block
+  const mapped = b.slice(0, 10).every(x => x === 0) && b[10] === 0xff && b[11] === 0xff
+  const nat64 = b[0] === 0 && b[1] === 0x64 && b[2] === 0xff && b[3] === 0x9b && b.slice(4, 12).every(x => x === 0)
+  if (mapped || nat64) return !v4Refused(b.slice(12, 16))
+  const hi = b[0]!
+  if (hi === 0xff) return false // ff00::/8
+  if ((hi & 0xfe) === 0xfc) return false // fc00::/7
+  if (hi === 0xfe && (b[1]! & 0xc0) >= 0x80) return false // fe80::/10 and fec0::/10
+  if (hi === 0x20 && b[1] === 0x01 && b[2] === 0x0d && b[3] === 0xb8) return false // 2001:db8::/32
+  return true
+}

package/src/infra/fetch/connect-retry.ts

@@ -0,0 +1,87 @@
+import { getBaseLogger } from '../../core/logging/index.js'
+
+export interface ConnectRetryOptions {
+  /** Waits between tries; the number of tries is delaysMs.length + 1. Default [5_000, 15_000, 40_000] (4 tries over ~60 s). */
+  delaysMs?: readonly number[]
+  /** Stops waiting and throws the last fault when aborted (checked before each try and during each wait). */
+  signal?: AbortSignal
+  /** Called before each wait with the fault, the 1-based try that failed, and the wait. */
+  onRetry?: (info: { error: unknown; attempt: number; delayMs: number }) => void
+}
+
+const DEFAULT_DELAYS: readonly number[] = [5_000, 15_000, 40_000]
+const MAX_DEPTH = 8
+
+const UNSENT_CODES = new Set(['UND_ERR_CONNECT_TIMEOUT', 'EAI_AGAIN', 'ENOTFOUND', 'ECONNREFUSED', 'ENETUNREACH', 'EHOSTUNREACH'])
+const SENT_CODES = new Set(['ECONNRESET', 'UND_ERR_SOCKET', 'EPIPE', 'UND_ERR_HEADERS_TIMEOUT', 'UND_ERR_BODY_TIMEOUT'])
+
+/** The first unsent-phase code found in the error's chain, or null when it is not an unsent fault. */
+function unsentCode(err: unknown): string | null {
+  const state: { found: string | null; sent: boolean } = { found: null, sent: false }
+  const seen = new Set<unknown>()
+  const walk = (node: unknown, depth: number): void => {
+    if (typeof node !== 'object' || node === null || depth > MAX_DEPTH || seen.has(node)) return
+    seen.add(node)
+    const { code, name, cause, errors } = node as { code?: unknown; name?: unknown; cause?: unknown; errors?: unknown }
+    if (name === 'AbortError' || name === 'TimeoutError') state.sent = true
+    if (typeof code === 'string') {
+      if (SENT_CODES.has(code)) state.sent = true
+      else if (UNSENT_CODES.has(code) && state.found === null) state.found = code
+    }
+    walk(cause, depth + 1)
+    if (Array.isArray(errors)) for (const e of errors) walk(e, depth + 1)
+  }
+  walk(err, 0)
+  return state.sent ? null : state.found
+}
+
+/** True when the fault happened while connecting (nothing was sent), so the request can be sent again safely. */
+export function isUnsentFault(err: unknown): boolean {
+  return unsentCode(err) !== null
+}
+
+const aborted = (signal: AbortSignal | undefined): boolean => signal?.aborted === true
+
+function wait(ms: number, signal: AbortSignal | undefined): Promise<void> {
+  return new Promise<void>((resolve) => {
+    if (signal?.aborted === true) { resolve(); return }
+    const done = (): void => { clearTimeout(timer); signal?.removeEventListener('abort', done); resolve() }
+    const timer = setTimeout(done, ms)
+    signal?.addEventListener('abort', done, { once: true })
+  })
+}
+
+/**
+ * Runs `send`, and sends again after a wait when it failed in the connect phase. Any other fault is thrown at once.
+ * When the tries run out, or `signal` aborts, the last fault is thrown unchanged.
+ */
+export async function retryUnsent<R>(send: () => Promise<R>, opts: ConnectRetryOptions = {}): Promise<R> {
+  const delays = opts.delaysMs ?? DEFAULT_DELAYS
+  for (let attempt = 1; ; attempt++) {
+    try {
+      return await send()
+    } catch (error) {
+      const code = unsentCode(error)
+      const delayMs = delays[attempt - 1]
+      if (code === null || delayMs === undefined || aborted(opts.signal)) throw error
+      getBaseLogger().warn({ code, attempt, delayMs }, 'connect failed before anything was sent; trying again')
+      opts.onRetry?.({ error, attempt, delayMs })
+      await wait(delayMs, opts.signal)
+      if (aborted(opts.signal)) throw error
+    }
+  }
+}
+
+/** `fetch` that is tried again only when the connection could not be made. A response of any status is returned. */
+export function fetchRetryingUnsent(
+  input: string | URL | Request,
+  init?: RequestInit,
+  opts: ConnectRetryOptions & { fetch?: typeof fetch } = {},
+): Promise<Response> {
+  const doFetch = opts.fetch ?? fetch
+  const signal = opts.signal ?? init?.signal ?? undefined
+  return retryUnsent(
+    () => doFetch(input instanceof Request ? input.clone() : input, init),
+    { ...opts, ...(signal !== undefined ? { signal } : {}) },
+  )
+}