pipework 0.254.0 → 0.255.0

package/src/infra/email/schema.ts

@@ -9,6 +9,7 @@ import { define } from '../../data/domain/index.js'
 import { field } from '../../data/domain/field.js'
 import { schema } from '../../data/schema/namespace.js'
 import { index, unique } from '../../data/db/idx/index.js'
+import { sql } from '../../data/db/sql.js'
 
 export const EMAIL_TABLE_NAMES: ReadonlySet<string> = new Set([
   'pipework_email_messages', 'pipework_email_suppressions', 'pipework_email_inbound', 'pipework_email_unsubscribes',
@@ -71,6 +72,8 @@ export const emailInbound = define('pipework_email_inbound', {
   rawLocation: field.jsonb(schema.check.json()),
   rawExpiresAt: field.timestamp(),
   rawDeletedAt: field.timestamp().nullable(),
+  // Set when `sweep` nulls the address, subject, recipients and Message-ID after `metadataRetentionDays`.
+  metadataScrubbedAt: field.timestamp().nullable(),
   status: field.enum(INBOUND_STATUSES).default('received'),
   error: field.text().nullable(),
   messageIdHeader: field.text().nullable(),
@@ -92,5 +95,6 @@ export const emailInbound = define('pipework_email_inbound', {
     bySource: unique('pipework_email_inbound_source_id_key').on(t['sourceId']),
     byMessageId: index('idx_pipework_email_inbound_message_id').on(t['messageIdHeader']),
     byExpiry: index('idx_pipework_email_inbound_raw_expires').on(t['rawExpiresAt']),
+    unscrubbed: index('idx_pipework_email_inbound_unscrubbed').on(t['receivedAt']).where(sql.raw('metadata_scrubbed_at IS NULL')),
   }),
 })

package/src/infra/email/sns.ts

@@ -94,7 +94,7 @@ const processEvent = __flowStep({ ref: 'pipework/email:notification' }, async (e
     } else {
       getBaseLogger().info({ event: 'email bounce not suppressed', address, bounceType: r.bounceType }, 'transient bounce: address kept')
     }
-    await auditEmail(r.kind, messageId, { address, bounceType: r.bounceType, suppressed: permanent, messageTraceId })
+    await auditEmail(r.kind, messageId, { bounceType: r.bounceType, suppressed: permanent, messageTraceId })
   }
 })
 

package/src/infra/outbox/core.ts

@@ -13,6 +13,9 @@ const LEASE_MS = 30_000
 const BATCH = 50
 const MAX_BATCHES = 20
 const MAX_BACKOFF_MS = 3_600_000
+const SWEEP_BATCH = 500
+const DEFAULT_SWEEP_LIMIT = 10_000
+const SWEEP_EVERY_MS = 3_600_000
 
 export interface OutboxTotals {
   delivered: number
@@ -61,6 +64,27 @@ export interface OutboxOptions<D> {
   readonly onGivenUp: (row: Record<string, unknown>, attempt: number, error: string) => Promise<void>
 }
 
+/** Retention for an outbox's rows, in days. Unset: kept. */
+export interface OutboxRetention {
+  /** Delivered rows (and their payloads) older than this are deleted. */
+  readonly deliveredRetentionDays: number
+  /** Given-up rows (`failed_at` set, never delivered) older than this are deleted. Unset: kept. */
+  readonly failedRetentionDays?: number
+}
+
+/** Refuses a retention that is not whole days of zero or more, naming `who` and the option. */
+export function checkRetention(who: string, config: { readonly deliveredRetentionDays?: number; readonly failedRetentionDays?: number }): OutboxRetention | undefined {
+  for (const name of ['deliveredRetentionDays', 'failedRetentionDays'] as const) {
+    const v = config[name]
+    if (v !== undefined && !(Number.isInteger(v) && v >= 0)) throw new Error(`[pipework] ${who}: ${name} must be an integer of zero or more, got ${String(v)}.`)
+  }
+  if (config.failedRetentionDays !== undefined && config.deliveredRetentionDays === undefined) {
+    throw new Error(`[pipework] ${who}: failedRetentionDays needs deliveredRetentionDays too.`)
+  }
+  if (config.deliveredRetentionDays === undefined) return undefined
+  return { deliveredRetentionDays: config.deliveredRetentionDays, ...(config.failedRetentionDays !== undefined ? { failedRetentionDays: config.failedRetentionDays } : {}) }
+}
+
 export interface Outbox {
   /** Delivers what is due now, up to 20 batches; rows that fail wait for the next call. */
   drain(): Promise<OutboxTotals>
@@ -70,6 +94,26 @@ export interface Outbox {
    * `prev_attempts`, the attempts it had when given up.
    */
   requeue(options: { sink?: string; failedSince?: Date }): Promise<Record<string, unknown>[]>
+  /**
+   * Deletes delivered rows whose `delivered_at` is older than `deliveredRetentionDays` and, with `failedRetentionDays`,
+   * given-up rows whose `failed_at` is older than that. A row still queued or being retried is never deleted.
+   * Deletes in batches of 500, at most `limit` rows per call (default 10 000).
+   */
+  sweep(options: OutboxRetention & { limit?: number }): Promise<{ deleted: number }>
+}
+
+/** A function that runs `outbox.sweep` with `retention` at most once an hour per process, and never throws; `onError` gets a failure. No retention: does nothing. */
+export function hourlySweep(outbox: Outbox, retention: OutboxRetention | undefined, onError: (err: unknown) => void): () => Promise<void> {
+  let last = 0
+  return async () => {
+    if (retention === undefined || Date.now() - last < SWEEP_EVERY_MS) return
+    last = Date.now()
+    try {
+      await outbox.sweep(retention)
+    } catch (err) {
+      onError(err)
+    }
+  }
 }
 
 function withTimeout(promise: Promise<void>, ms: number): Promise<void> {
@@ -136,7 +180,33 @@ export function createOutbox<D>(options: OutboxOptions<D>): Outbox {
     }))
   }
 
+  async function deleteBatch(predicate: ReturnType<typeof sql>, size: number): Promise<number> {
+    const rows = await db().execute(sql`
+      DELETE FROM ${table} WHERE id IN (SELECT id FROM ${table} WHERE ${predicate} LIMIT ${size}) RETURNING id`)
+    return rows.length
+  }
+
   return {
+    async sweep(opts) {
+      const days = [['deliveredRetentionDays', opts.deliveredRetentionDays], ['failedRetentionDays', opts.failedRetentionDays], ['limit', opts.limit]] as const
+      for (const [name, v] of days) {
+        if (v !== undefined && !(Number.isInteger(v) && v >= (name === 'limit' ? 1 : 0))) {
+          throw new ValidationError({ [name]: ['must be a whole number' + (name === 'limit' ? ' of one or more' : ' of zero or more')] }, `Outbox sweep ${name} is invalid.`)
+        }
+      }
+      const delivered = sql`delivered_at IS NOT NULL AND delivered_at < clock_timestamp() - ${opts.deliveredRetentionDays} * interval '1 day'`
+      const failed = opts.failedRetentionDays === undefined ? null
+        : sql`delivered_at IS NULL AND failed_at IS NOT NULL AND failed_at < clock_timestamp() - ${opts.failedRetentionDays} * interval '1 day'`
+      const predicate = failed === null ? delivered : sql`((${delivered}) OR (${failed}))`
+      const limit = opts.limit ?? DEFAULT_SWEEP_LIMIT
+      let deleted = 0
+      while (deleted < limit) {
+        const n = await deleteBatch(predicate, Math.min(SWEEP_BATCH, limit - deleted))
+        deleted += n
+        if (n === 0) break
+      }
+      return { deleted }
+    },
     async requeue(opts) {
       if (opts.failedSince !== undefined && !(opts.failedSince instanceof Date && !Number.isNaN(opts.failedSince.getTime()))) {
         throw new ValidationError({ failedSince: ['must be a valid Date'] }, 'Outbox failedSince must be a Date.')

package/src/infra/support/handle.ts

@@ -17,7 +17,7 @@ import { recordBeat, beatsEnabled } from '../alerts/beat.js'
 import type { Mail } from '../email/mail.js'
 import { sanitizeHtml, htmlToText } from '../email/sanitize.js'
 import type { ReceivedEmail } from '../email/inbound.js'
-import { createOutbox, type Sink, type SinkDelivery } from '../outbox/core.js'
+import { checkRetention, createOutbox, hourlySweep, type Sink, type SinkDelivery } from '../outbox/core.js'
 import { TICKET_STATES } from './schema.js'
 import { describeError } from '../../core/logging/error-chain.js'
 
@@ -99,6 +99,14 @@ export interface SupportConfig {
   readonly maxAttempts?: number
   /** First retry delay, doubling per attempt up to one hour; default 5000. */
   readonly backoffMs?: number
+  /**
+   * Days a delivered event row, with its payload (the ticket subject), is kept before the sweep deletes it. Unset: kept.
+   * `tick` (so `support.job`) runs the sweep at most once an hour per process; `sweep()` runs it on demand.
+   * The ticket audit rows carry ids only; they expire with the trace partitions (`auditRetentionDays`).
+   */
+  readonly deliveredRetentionDays?: number
+  /** Days an event row that was given up on (never delivered) is kept. Needs `deliveredRetentionDays`. Unset: kept. */
+  readonly failedRetentionDays?: number
 }
 
 export interface Ticket {
@@ -256,6 +264,8 @@ export interface Support extends WithFor<TicketApi> {
   readonly job: Handler<Record<string, unknown>, unknown>
   /** What one run of `job` does; callable from a test. Needs a Flow. */
   tick(): Promise<{ delivered: number; retried: number; failed: number }>
+  /** Deletes event rows past `deliveredRetentionDays` / `failedRetentionDays` now, up to `limit` (default 10 000); `deleted: 0` when neither is set. Needs a Flow. */
+  sweep(options?: { limit?: number }): Promise<{ deleted: number }>
   /**
    * Puts the event deliveries that were given up (and never delivered) back on the queue, for every tenant, optionally
    * only those of `sink` and/or that failed at or after `failedSince`; the next `tick` sends them. Delivery ids stay the
@@ -358,6 +368,7 @@ export function createSupport(config: SupportConfig): Support {
   if ((joinThread as string) !== 'contact' && (joinThread as string) !== 'conversation') {
     throw new Error(`[pipework] support.create: joinThread must be 'contact' or 'conversation', got ${JSON.stringify(joinThread)}.`)
   }
+  const retention = checkRetention('support.create', config)
   const jobType = config.jobType ?? 'pipework.support.events'
   const replyTemplate = config.replyTemplate ?? 'pipework.support.reply'
   const distinctApprover = config.distinctApprover ?? true
@@ -545,7 +556,7 @@ export function createSupport(config: SupportConfig): Support {
         RETURNING *`)
       ticket = toTicket(rows[0]!)
       created = true
-      await audit(flow, tx, tenantId, 'create', ticket.id, { subject: ticket.subject, contact: ticket.contactEmail, channel: m.channel, userId: ticket.userId })
+      await audit(flow, tx, tenantId, 'create', ticket.id, { channel: m.channel, userId: ticket.userId })
     } else {
       await discardLive(flow, tx, ticket, 'customer-replied')
       if (ticket.state !== 'open') ticket = await moveTo(flow, tx, ticket, 'open', true)
@@ -802,7 +813,15 @@ export function createSupport(config: SupportConfig): Support {
     await audit(flow, null, payload.tenantId, verb, row['ticket_id'] as string, { sink: row['sink'], kind: row['kind'], ...extra })
   }
 
-  const drainStep = __flowStep({ ref: 'pipework/support:tick' }, () => outbox.drain())
+  const sweepIfDue = hourlySweep(outbox, retention, (err) => {
+    getBaseLogger().error({ err }, 'support event sweep failed')
+    void recordFault('support-event-sweep', faultMessage(err), {})
+  })
+  const drainStep = __flowStep({ ref: 'pipework/support:tick' }, async () => {
+    const totals = await outbox.drain()
+    await sweepIfDue()
+    return totals
+  })
   // A beat for `alerts.source.loopStalled`, written only where the blueprint has `alerts`.
   const beat = async (outcome: { ok: boolean; error?: string }): Promise<void> => {
     const ctx = getContext()
@@ -855,6 +874,10 @@ export function createSupport(config: SupportConfig): Support {
     },
     job: job as unknown as Handler<Record<string, unknown>, unknown>,
     tick: () => tickStep(),
+    async sweep(options = {}) {
+      requireContext()
+      return retention === undefined ? { deleted: 0 } : outbox.sweep({ ...retention, ...(options.limit !== undefined ? { limit: options.limit } : {}) })
+    },
     async requeueFailed(options) {
       requireContext()
       nonEmpty('by', options.by)