pipework 0.252.1 → 0.254.0

package/src/infra/email/sanitize.ts

@@ -0,0 +1,284 @@
+/**
+ * Allow-list HTML sanitiser for mail from strangers. The input is tokenised and the output is built only from
+ * allow-listed tags and attributes, every tag re-serialised and every text run and attribute value escaped.
+ * No slice of the input is copied through. Output is balanced and `sanitizeHtml(sanitizeHtml(x)) === sanitizeHtml(x)`.
+ */
+
+const VOID = new Set(['br', 'hr', 'img', 'col'])
+const ALLOWED = new Set([
+  'p', 'br', 'div', 'span', 'b', 'strong', 'i', 'em', 'u', 's', 'strike', 'sub', 'sup', 'small', 'blockquote', 'pre', 'code',
+  'ul', 'ol', 'li', 'dl', 'dt', 'dd', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'table', 'thead', 'tbody', 'tfoot', 'tr', 'th', 'td',
+  'caption', 'colgroup', 'col', 'a', 'img', 'font', 'center',
+])
+/**
+ * Dropped together with everything inside them. Wrappers such as form, button, select, option and head are not listed:
+ * they are unwrapped (tag gone, text kept) and their dangerous children are dropped one by one.
+ */
+const DROPPED = new Set([
+  'script', 'style', 'iframe', 'frame', 'frameset', 'object', 'embed', 'applet', 'input', 'textarea',
+  'svg', 'math', 'template', 'noscript', 'title', 'base', 'link', 'meta',
+])
+/** Dropped elements whose content is not markup: skipped to the matching end tag. */
+const RAW_TEXT = new Set(['script', 'style', 'iframe', 'textarea', 'template', 'noscript', 'title'])
+/** Dropped elements with no content. */
+const EMPTY_DROPPED = new Set(['input', 'base', 'link', 'meta', 'embed', 'frame'])
+
+const NAMED: Record<string, string> = {
+  amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', colon: ':', Tab: '\t', NewLine: '\n', copy: '©',
+  reg: '®', hellip: '…', mdash: '—', ndash: '–', lsquo: '‘', rsquo: '’', ldquo: '“',
+  rdquo: '”', bull: '•', middot: '·', euro: '€', zwnj: '', zwj: '', shy: '', trade: '\u2122',
+  rarr: '\u2192', larr: '\u2190', times: '\u00d7', deg: '\u00b0', laquo: '\u00ab', raquo: '\u00bb', pound: '\u00a3',
+  yen: '\u00a5', cent: '\u00a2', sect: '\u00a7', para: '\u00b6',
+}
+
+/** Marks an unknown but well-formed named reference in decoded text; `sanitizeHtml` passes it through, `htmlToText` drops it. */
+const REF_OPEN = '\ue000'
+const REF_CLOSE = '\ue001'
+const MARKERS = /[\ue000\ue001]/g
+
+/** In text mode an unknown reference like `&foo;` is wrapped in markers instead of left bare. */
+function decodeEntities(s: string, text = false): string {
+  if (text) s = s.replace(MARKERS, '')
+  if (!s.includes('&')) return s
+  return s.replace(/&(#[xX][0-9a-fA-F]+|#[0-9]+|[A-Za-z][A-Za-z0-9]*);?/g, (match, body: string) => {
+    if (body.startsWith('#')) {
+      const code = body[1] === 'x' || body[1] === 'X' ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10)
+      if (!Number.isFinite(code) || code <= 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff) || code === 0xe000 || code === 0xe001) return '�'
+      return String.fromCodePoint(code)
+    }
+    const value = match.endsWith(';') ? NAMED[body] : undefined
+    if (value !== undefined) return value
+    return text && match.endsWith(';') && body.length >= 2 && body.length <= 32 ? `${REF_OPEN}${match}${REF_CLOSE}` : match
+  })
+}
+
+function escapeText(s: string): string {
+  return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;')
+}
+
+/** Escapes a decoded text run, leaving the marked unknown references as written (they cannot form markup). */
+function escapeRun(s: string): string {
+  return escapeText(s).replace(/\ue000&amp;([A-Za-z][A-Za-z0-9]*;)\ue001/g, '&$1').replace(MARKERS, '')
+}
+
+function escapeAttr(s: string): string {
+  return escapeText(s).replace(/"/g, '&quot;').replace(/'/g, '&#39;')
+}
+
+type Token =
+  | { readonly t: 'text'; readonly v: string }
+  | { readonly t: 'start'; readonly name: string; readonly attrs: ReadonlyMap<string, string> }
+  | { readonly t: 'end'; readonly name: string }
+
+interface Tag { readonly end: number; readonly name: string; readonly closing: boolean; readonly selfClosing: boolean; readonly attrs: Map<string, string> }
+
+const isSpace = (c: string | undefined): boolean => c === ' ' || c === '\t' || c === '\n' || c === '\r' || c === '\f'
+const isLetter = (c: string | undefined): boolean => c !== undefined && /[A-Za-z]/.test(c)
+
+/** Reads the tag starting at `<`; null when it never closes. */
+function readTag(s: string, start: number): Tag | null {
+  const n = s.length
+  let j = start + 1
+  const closing = s[j] === '/'
+  if (closing) j++
+  const nameStart = j
+  while (j < n && !isSpace(s[j]) && s[j] !== '/' && s[j] !== '>') j++
+  const name = s.slice(nameStart, j).toLowerCase()
+  const attrs = new Map<string, string>()
+  for (;;) {
+    while (j < n && (isSpace(s[j]) || s[j] === '/')) j++
+    if (j >= n) return null
+    if (s[j] === '>') return { end: j + 1, name, closing, selfClosing: s[j - 1] === '/', attrs }
+    const aStart = j
+    while (j < n && !isSpace(s[j]) && s[j] !== '=' && s[j] !== '/' && s[j] !== '>') j++
+    if (j === aStart) { j++; continue }
+    const attr = s.slice(aStart, j).toLowerCase()
+    while (j < n && isSpace(s[j])) j++
+    let value = ''
+    if (s[j] === '=') {
+      j++
+      while (j < n && isSpace(s[j])) j++
+      const q = s[j]
+      if (q === '"' || q === "'") {
+        const k = s.indexOf(q, j + 1)
+        if (k < 0) return null
+        value = s.slice(j + 1, k)
+        j = k + 1
+      } else {
+        const vStart = j
+        while (j < n && !isSpace(s[j]) && s[j] !== '>') j++
+        value = s.slice(vStart, j)
+      }
+    }
+    if (!attrs.has(attr)) attrs.set(attr, decodeEntities(value))
+  }
+}
+
+/** Tokens of the input with dropped elements and their content already removed. Text and attribute values are decoded. */
+function* tokenize(s: string): Generator<Token> {
+  const n = s.length
+  let i = 0
+  let skip: { name: string; depth: number } | null = null
+  while (i < n) {
+    if (s[i] !== '<') {
+      const k = s.indexOf('<', i)
+      const end = k < 0 ? n : k
+      if (skip === null) yield { t: 'text', v: decodeEntities(s.slice(i, end), true) }
+      i = end
+      continue
+    }
+    const next = s[i + 1]
+    if (isLetter(next) || (next === '/' && isLetter(s[i + 2]))) {
+      const tag = readTag(s, i)
+      if (tag === null) return
+      i = tag.end
+      if (skip !== null) {
+        if (tag.name === skip.name) {
+          if (tag.closing) { if (--skip.depth === 0) skip = null } else if (!tag.selfClosing) skip.depth++
+        }
+        continue
+      }
+      if (tag.closing) { yield { t: 'end', name: tag.name }; continue }
+      if (DROPPED.has(tag.name)) {
+        if (EMPTY_DROPPED.has(tag.name) || tag.selfClosing) continue
+        if (RAW_TEXT.has(tag.name)) {
+          const close = new RegExp(`</${tag.name}[\\s/>]`, 'ig')
+          close.lastIndex = i
+          const m = close.exec(s)
+          if (m === null) return
+          const gt = s.indexOf('>', m.index)
+          if (gt < 0) return
+          i = gt + 1
+        } else skip = { name: tag.name, depth: 1 }
+        continue
+      }
+      yield { t: 'start', name: tag.name, attrs: tag.attrs }
+      continue
+    }
+    if (next === '!' || next === '?' || next === '/') {
+      let end: number
+      if (s.startsWith('<!-->', i)) end = i + 5
+      else if (s.startsWith('<!--->', i)) end = i + 6
+      else if (s.startsWith('<!--', i)) { const k = s.indexOf('-->', i + 4); end = k < 0 ? -1 : k + 3 }
+      else if (s.startsWith('<![CDATA[', i)) { const k = s.indexOf(']]>', i + 9); end = k < 0 ? -1 : k + 3 }
+      else { const k = s.indexOf('>', i + 2); end = k < 0 ? -1 : k + 1 }
+      if (end < 0) return
+      i = end
+      continue
+    }
+    // A lone '<' is text.
+    const k = s.indexOf('<', i + 1)
+    const end = k < 0 ? n : k
+    if (skip === null) yield { t: 'text', v: decodeEntities(s.slice(i, end), true) }
+    i = end
+  }
+}
+
+const cleanUrl = (v: string): string => v.replace(/[\u0000-\u001f\u007f-\u009f]/g, '').trim()
+const NUM = /^\d{1,5}$/
+const SIZE = /^\d{1,5}%?$/
+const ALIGN = /^(left|right|center|justify)$/i
+const VALIGN = /^(top|middle|bottom|baseline)$/i
+
+const ATTRS: Record<string, Record<string, RegExp | 'text'>> = {
+  a: { title: 'text' },
+  img: { alt: 'text', width: SIZE, height: SIZE, title: 'text' },
+  td: { colspan: NUM, rowspan: NUM, align: ALIGN, valign: VALIGN },
+  th: { colspan: NUM, rowspan: NUM, align: ALIGN, valign: VALIGN },
+  table: { border: NUM, cellpadding: NUM, cellspacing: NUM, width: SIZE, align: ALIGN },
+  col: { span: NUM, width: SIZE },
+  colgroup: { span: NUM, width: SIZE },
+  ol: { start: NUM, type: /^[1aAiI]$/ },
+  font: { color: /^#?[0-9A-Za-z]{3,20}$/, size: /^[+-]?\d$/, face: /^[\w ,-]{1,100}$/ },
+  p: { align: ALIGN }, div: { align: ALIGN },
+  h1: { align: ALIGN }, h2: { align: ALIGN }, h3: { align: ALIGN }, h4: { align: ALIGN }, h5: { align: ALIGN }, h6: { align: ALIGN },
+}
+
+function safeHref(attrs: ReadonlyMap<string, string>): string | null {
+  const href = cleanUrl(attrs.get('href') ?? '')
+  return /^(https?:|mailto:)/i.test(href) ? href : null
+}
+
+function serialiseStart(name: string, attrs: ReadonlyMap<string, string>): string {
+  let out = `<${name}`
+  if (name === 'a') {
+    const href = safeHref(attrs)
+    if (href !== null) out += ` href="${escapeAttr(href)}"`
+  }
+  const rules = ATTRS[name]
+  if (rules !== undefined) {
+    for (const [attr, rule] of Object.entries(rules)) {
+      const v = attrs.get(attr)
+      if (v === undefined) continue
+      if (rule === 'text' || rule.test(v.trim())) out += ` ${attr}="${escapeAttr(rule === 'text' ? v : v.trim())}"`
+    }
+  }
+  if (name === 'a') out += ' rel="noopener noreferrer nofollow" target="_blank"'
+  return out + '>'
+}
+
+/** Returns the HTML reduced to allow-listed tags and attributes: no script, event handlers, style, forms or remote resources. */
+export function sanitizeHtml(html: string): string {
+  let out = ''
+  const open: string[] = []
+  for (const tok of tokenize(html)) {
+    if (tok.t === 'text') { out += escapeRun(tok.v); continue }
+    if (!ALLOWED.has(tok.name)) continue
+    if (tok.t === 'start') {
+      if (tok.name === 'img') {
+        const src = cleanUrl(tok.attrs.get('src') ?? '')
+        if (/^cid:/i.test(src)) {
+          out += `<img src="${escapeAttr(src)}"${serialiseStart('img', tok.attrs).slice(4, -1)}>`
+        } else {
+          const alt = tok.attrs.get('alt') ?? ''
+          if (alt.trim() !== '') out += escapeText(alt)
+        }
+        continue
+      }
+      out += serialiseStart(tok.name, tok.attrs)
+      if (!VOID.has(tok.name)) open.push(tok.name)
+    } else {
+      const at = open.lastIndexOf(tok.name)
+      if (at < 0 || VOID.has(tok.name)) continue
+      while (open.length > at) out += `</${open.pop()}>`
+    }
+  }
+  while (open.length > 0) out += `</${open.pop()}>`
+  return out
+}
+
+const BLOCK = new Set(['p', 'div', 'br', 'li', 'tr', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'blockquote', 'ul', 'ol', 'table', 'pre', 'hr'])
+
+const LINE_START_ONLY = new Set(['li', 'tr', 'br'])
+
+/** The readable text of the HTML: block tags become line breaks, entities are decoded, script and style are excluded. */
+export function htmlToText(html: string): string {
+  let out = ''
+  let pre = 0
+  const links: { href: string; at: number }[] = []
+  for (const tok of tokenize(html)) {
+    if (tok.t === 'text') {
+      const v = tok.v.replace(/\ue000[^\ue001]*\ue001/g, '')
+      out += pre > 0 ? v : v.replace(/\s+/g, ' ')
+    } else if (tok.t === 'start') {
+      if (tok.name === 'pre') pre++
+      if (tok.name === 'a') {
+        const href = safeHref(tok.attrs)
+        if (href !== null) links.push({ href, at: out.length })
+      }
+      if (BLOCK.has(tok.name)) out += '\n'
+    } else {
+      if (tok.name === 'pre' && pre > 0) pre--
+      if (tok.name === 'a') {
+        const link = links.pop()
+        if (link !== undefined) {
+          const label = out.slice(link.at).trim()
+          if (label !== '' && label !== link.href && `mailto:${label}` !== link.href) out += ` (${link.href})`
+        }
+      }
+      if (BLOCK.has(tok.name) && !LINE_START_ONLY.has(tok.name)) out += '\n'
+      else if (tok.name === 'td' || tok.name === 'th') out += ' '
+    }
+  }
+  return out.split('\n').map(l => (pre > 0 ? l : l.replace(/[ \t ]+/g, ' ').trim())).join('\n').replace(/\n{3,}/g, '\n\n').trim()
+}

package/src/infra/support/handle.ts

@@ -15,6 +15,7 @@ import type { Handler } from '../../request/di/types.js'
 import { __flowStep } from '../../trace/runtime/flow-step.js'
 import { recordBeat, beatsEnabled } from '../alerts/beat.js'
 import type { Mail } from '../email/mail.js'
+import { sanitizeHtml, htmlToText } from '../email/sanitize.js'
 import type { ReceivedEmail } from '../email/inbound.js'
 import { createOutbox, type Sink, type SinkDelivery } from '../outbox/core.js'
 import { TICKET_STATES } from './schema.js'
@@ -64,7 +65,8 @@ export interface SupportConfig {
   /**
    * Decides what `receiveEmail` does with a message. Default: virus verdict `FAIL` drops it; DMARC `FAIL`, or SPF and
    * DKIM both `FAIL`, separates it (the From address is probably forged, so it must not add text to a ticket an agent
-   * is reading: it always opens a new ticket); anything else, including no verdicts (the folder stand-in), is accepted.
+   * is reading: it always opens a new ticket); anything else, including no verdicts (the folder stand-in), is accepted. A message it
+   * separates never names a user: `resolveContact` is not called for it.
    */
   readonly screen?: (email: ReceivedEmail) => ScreenVerdict
   /**
@@ -72,8 +74,16 @@ export interface SupportConfig {
    * to that tenant; a throw rolls the receive back) when a message has no user of its own: for a new ticket, and for a
    * message joining a ticket whose `user_id` is null, which it fills. A ticket's non-null `user_id` is never changed.
    * `receive` with `contact.userId` does not call it. Null: no known user.
+   *
+   * `email` is the received message when it came through `receiveEmail` (its `verdicts` say whether the From can be
+   * believed) and `null` for `receive` (chat/api). A message `screen` separates (default: DMARC `FAIL`, or SPF and DKIM
+   * both `FAIL`) never reaches this hook: its ticket's `user_id` stays null. To also require a pass:
+   * `resolveContact: async ({ address, email }, tx) => email !== null && email.verdicts?.dmarc !== 'PASS' ? null : lookUp(address, tx)`.
    */
-  readonly resolveContact?: (contact: { address: string; name: string | null; tenantId: string }, tx: DB) => Promise<{ userId: string } | null>
+  readonly resolveContact?: (
+    contact: { address: string; name: string | null; tenantId: string; email: ReceivedEmail | null },
+    tx: DB,
+  ) => Promise<{ userId: string } | null>
   /**
    * Who may join a ticket by thread. `'contact'` (default): only the ticket's contact; a message from anyone else in the
    * thread opens its own ticket, and `receive` with a `ticketId` refuses a sender who is not the contact. `'conversation'`:
@@ -278,11 +288,17 @@ function toTicket(r: Record<string, unknown>): Ticket {
   }
 }
 
+/** Inbound html is stranger-sent: rows stored before sanitising at ingest are cleaned on read (idempotent). */
+function storedHtml(r: Record<string, unknown>): string | null {
+  const html = (r['html'] ?? null) as string | null
+  return html !== null && r['kind'] === 'inbound' ? sanitizeHtml(html) : html
+}
+
 function toMessage(r: Record<string, unknown>): TicketMessage {
   return {
     id: r['id'] as string, ticketId: r['ticket_id'] as string, kind: r['kind'] as TicketMessage['kind'],
     channel: r['channel'] as TicketMessage['channel'], subject: (r['subject'] ?? null) as string | null,
-    text: r['text'] as string, html: (r['html'] ?? null) as string | null, fromAddress: (r['from_address'] ?? null) as string | null,
+    text: r['text'] as string, html: storedHtml(r), fromAddress: (r['from_address'] ?? null) as string | null,
     inboundEmailId: (r['inbound_email_id'] ?? null) as string | null,
     attachments: (r['attachments'] ?? null) as TicketMessage['attachments'],
     draftStatus: (r['draft_status'] ?? null) as TicketMessage['draftStatus'], version: Number(r['version']),
@@ -474,8 +490,10 @@ export function createSupport(config: SupportConfig): Support {
     readonly conversationId: string | null
     readonly attachments: unknown
     readonly ticketId: string | null
-    /** Never add to an existing ticket. */
+    /** Never add to an existing ticket, never name a user. */
     readonly separate?: boolean
+    /** The received message; null for chat/api. */
+    readonly email?: ReceivedEmail | null
   }
 
   async function ingest(flow: Flow, tx: DB, tenantId: string, m: Incoming): Promise<ReceiveResult> {
@@ -503,8 +521,12 @@ export function createSupport(config: SupportConfig): Support {
       if (found[0] !== undefined) ticket = await loadTicket(tx, tenantId, found[0]['id'] as string, true)
     }
 
-    const resolve = async (): Promise<string | null> =>
-      m.userId ?? (await config.resolveContact?.({ address: m.contactEmail, name: m.contactName, tenantId }, tx))?.userId ?? null
+    const resolve = async (): Promise<string | null> => {
+      if (m.userId !== null) return m.userId
+      if (m.separate === true) return null
+      const found = await config.resolveContact?.({ address: m.contactEmail, name: m.contactName, tenantId, email: m.email ?? null }, tx)
+      return found?.userId ?? null
+    }
 
     const joinedBy = ticket !== null && ticket.contactEmail !== m.contactEmail ? 'conversation' : null
     let created = false
@@ -534,9 +556,11 @@ export function createSupport(config: SupportConfig): Support {
       ticket = toTicket(rows[0]!)
     }
 
+    const html = m.html === null ? null : sanitizeHtml(m.html)
+    const text = m.text.trim() === '' && m.html !== null ? htmlToText(m.html) : m.text
     await tx.execute(sql`
       INSERT INTO support_message (tenant_id, ticket_id, kind, channel, text, html, from_address, inbound_email_id, attachments)
-      VALUES (${tenantId}::uuid, ${ticket.id}::uuid, 'inbound', ${m.channel}, ${m.text}, ${m.html}, ${m.fromAddress},
+      VALUES (${tenantId}::uuid, ${ticket.id}::uuid, 'inbound', ${m.channel}, ${text}, ${html}, ${m.fromAddress},
         ${m.inboundEmailId}::uuid, ${m.attachments === null ? null : JSON.stringify(m.attachments)}::jsonb)`)
     await audit(flow, tx, tenantId, 'receive', ticket.id, { channel: m.channel, emailId: m.inboundEmailId, created, separated: m.separate === true, ...(joinedBy === null ? {} : { joinedBy }), ...(userIdSet === null ? {} : { userIdSet }) })
     await emit(tx, created ? 'ticket.created' : 'ticket.customer-replied', ticket)
@@ -566,7 +590,7 @@ export function createSupport(config: SupportConfig): Support {
       channel: 'email', contactEmail, contactName: email.from.name === '' ? null : email.from.name, userId: null,
       subject: email.subject.trim() === '' ? '(no subject)' : email.subject, text: email.text ?? '', html: email.html,
       fromAddress: contactEmail, inboundEmailId: email.id, conversationId: email.conversationId,
-      attachments: attachments.length === 0 ? null : attachments, ticketId: null, separate: verdict === 'separate',
+      attachments: attachments.length === 0 ? null : attachments, ticketId: null, separate: verdict === 'separate', email,
     }))
   }