pipework 0.254.0 → 0.256.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/dist/infra/alerts/handle.d.ts +10 -0
- package/dist/infra/alerts/handle.d.ts.map +1 -1
- package/dist/infra/alerts/handle.js +9 -2
- package/dist/infra/alerts/handle.js.map +1 -1
- package/dist/infra/email/inbound.d.ts +12 -1
- package/dist/infra/email/inbound.d.ts.map +1 -1
- package/dist/infra/email/inbound.js +22 -1
- package/dist/infra/email/inbound.js.map +1 -1
- package/dist/infra/email/schema.d.ts +5 -0
- package/dist/infra/email/schema.d.ts.map +1 -1
- package/dist/infra/email/schema.js +4 -0
- package/dist/infra/email/schema.js.map +1 -1
- package/dist/infra/email/sns.js +1 -1
- package/dist/infra/email/sns.js.map +1 -1
- package/dist/infra/outbox/core.d.ts +24 -0
- package/dist/infra/outbox/core.d.ts.map +1 -1
- package/dist/infra/outbox/core.js +58 -0
- package/dist/infra/outbox/core.js.map +1 -1
- package/dist/infra/support/handle.d.ts +30 -4
- package/dist/infra/support/handle.d.ts.map +1 -1
- package/dist/infra/support/handle.js +47 -8
- package/dist/infra/support/handle.js.map +1 -1
- package/dist/infra/support/namespace.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/infra/alerts/handle.ts +16 -2
- package/src/infra/email/inbound.ts +34 -2
- package/src/infra/email/schema.ts +4 -0
- package/src/infra/email/sns.ts +1 -1
- package/src/infra/outbox/core.ts +70 -0
- package/src/infra/support/handle.ts +66 -11
|
@@ -9,7 +9,7 @@ import { __flowStep } from '../../trace/runtime/flow-step.js'
|
|
|
9
9
|
import { auditAlert } from './audit.js'
|
|
10
10
|
import { ALERT_SEVERITIES, ALERT_STATUSES } from './schema.js'
|
|
11
11
|
import { encodeCursor, decodeCursor } from '../../request/fixture/paginate.js'
|
|
12
|
-
import { createOutbox } from '../outbox/core.js'
|
|
12
|
+
import { checkRetention, createOutbox, hourlySweep } from '../outbox/core.js'
|
|
13
13
|
import type { AlertDelivery, AlertPayload, AlertSink } from './sinks.js'
|
|
14
14
|
import type { AlertSource } from './sources.js'
|
|
15
15
|
import { redactJson, redactText } from './redact.js'
|
|
@@ -42,6 +42,10 @@ export interface AlertsConfig {
|
|
|
42
42
|
readonly backoffMs?: number
|
|
43
43
|
/** How long an occurrence is kept; the tick deletes older ones. Default 30 days. */
|
|
44
44
|
readonly occurrenceRetentionMs?: number
|
|
45
|
+
/** Days a delivered delivery row, with its payload, is kept before the tick's sweep deletes it (at most once an hour per process). Unset: kept. */
|
|
46
|
+
readonly deliveredRetentionDays?: number
|
|
47
|
+
/** Days a delivery row that was given up on (never delivered) is kept. Needs `deliveredRetentionDays`. Unset: kept. */
|
|
48
|
+
readonly failedRetentionDays?: number
|
|
45
49
|
/**
|
|
46
50
|
* Send alerts of these severities in one periodic `digest` delivery per sink instead of one at a time. A raise or
|
|
47
51
|
* resolve of such an alert queues nothing (so `resendAfterMs` has no effect on it); each tick at least `everyMs`
|
|
@@ -208,6 +212,8 @@ export interface Alerts {
|
|
|
208
212
|
readonly job: Handler<Record<string, unknown>, unknown>
|
|
209
213
|
/** What one run of `job` does; callable from a test. Needs a Flow. */
|
|
210
214
|
tick(): Promise<TickResult>
|
|
215
|
+
/** Deletes delivery rows past `deliveredRetentionDays` / `failedRetentionDays` now, up to `limit` (default 10 000); `deleted: 0` when neither is set. */
|
|
216
|
+
sweep(options?: { limit?: number }): Promise<{ deleted: number }>
|
|
211
217
|
open(): Promise<Alert[]>
|
|
212
218
|
get(key: string): Promise<Alert | null>
|
|
213
219
|
deliveries(options?: { key?: string }): Promise<DeliveryRow[]>
|
|
@@ -338,6 +344,7 @@ export function createAlerts(config: AlertsConfig): Alerts {
|
|
|
338
344
|
if (config.digest.severities?.length === 0) throw new Error('[pipework] alerts.create: digest.severities needs at least one severity, or leave it out for warning.')
|
|
339
345
|
}
|
|
340
346
|
const digestSeverities = pgTextArray(config.digest === undefined ? [] : [...new Set(config.digest.severities ?? ['warning'])])
|
|
347
|
+
const retention = checkRetention('alerts.create', config)
|
|
341
348
|
const jobType = config.jobType ?? 'pipework.alerts'
|
|
342
349
|
const maxAttempts = config.maxAttempts ?? 10
|
|
343
350
|
const backoffMs = config.backoffMs ?? 5000
|
|
@@ -718,12 +725,18 @@ export function createAlerts(config: AlertsConfig): Alerts {
|
|
|
718
725
|
onGivenUp: (row, attempt, error) => auditAlert('deliver-failed', row['alert_key'] as string, { sink: row['sink'], kind: row['kind'], attempt, error }, database),
|
|
719
726
|
})
|
|
720
727
|
|
|
728
|
+
const sweepIfDue = hourlySweep(outbox, retention, (err) => {
|
|
729
|
+
log.error({ err }, 'alert delivery sweep failed')
|
|
730
|
+
})
|
|
731
|
+
|
|
721
732
|
async function runTick(): Promise<TickResult> {
|
|
722
733
|
let resolved = await autoResolve()
|
|
723
734
|
resolved += await runSources()
|
|
724
735
|
await pruneOccurrences()
|
|
725
736
|
await sendDigest()
|
|
726
|
-
|
|
737
|
+
const totals = await outbox.drain()
|
|
738
|
+
await sweepIfDue()
|
|
739
|
+
return { resolved, ...totals }
|
|
727
740
|
}
|
|
728
741
|
|
|
729
742
|
const flowTick = __flowStep({ ref: 'pipework/alerts:tick' }, runTick)
|
|
@@ -751,6 +764,7 @@ export function createAlerts(config: AlertsConfig): Alerts {
|
|
|
751
764
|
health,
|
|
752
765
|
job: job as unknown as Handler<Record<string, unknown>, unknown>,
|
|
753
766
|
tick: () => tickStep(),
|
|
767
|
+
sweep: async (options = {}) => retention === undefined ? { deleted: 0 } : outbox.sweep({ ...retention, ...(options.limit !== undefined ? { limit: options.limit } : {}) }),
|
|
754
768
|
async requeueFailed(options) {
|
|
755
769
|
if (typeof options.by !== 'string' || options.by.trim() === '') throw new ValidationError({ by: ['must be a non-empty string'] }, 'alerts.requeueFailed needs by, who is requeueing.')
|
|
756
770
|
const by = options.by.trim()
|
|
@@ -116,6 +116,8 @@ export interface InboundRecord {
|
|
|
116
116
|
readonly processedAt: Date | null
|
|
117
117
|
readonly rawExpiresAt: Date
|
|
118
118
|
readonly rawDeletedAt: Date | null
|
|
119
|
+
/** When `sweep` nulled the address, subject, recipients and Message-ID; null until then. */
|
|
120
|
+
readonly metadataScrubbedAt: Date | null
|
|
119
121
|
}
|
|
120
122
|
|
|
121
123
|
export interface InboundSweepResult {
|
|
@@ -123,6 +125,8 @@ export interface InboundSweepResult {
|
|
|
123
125
|
readonly deleted: number
|
|
124
126
|
/** Raw copies that could not be deleted (left for the next sweep). */
|
|
125
127
|
readonly failed: number
|
|
128
|
+
/** Rows whose address, subject, recipients and Message-ID were nulled (`metadataRetentionDays`). */
|
|
129
|
+
readonly scrubbed: number
|
|
126
130
|
}
|
|
127
131
|
|
|
128
132
|
export interface InboundScanResult {
|
|
@@ -253,6 +257,13 @@ export interface InboundConfig {
|
|
|
253
257
|
readonly resolveTenant?: (email: UnthreadedEmail) => Promise<string | null>
|
|
254
258
|
/** Days the raw copy is kept before `sweep` deletes it. Default 30. */
|
|
255
259
|
readonly rawRetentionDays?: number
|
|
260
|
+
/**
|
|
261
|
+
* Days after receipt that `sweep` nulls a message's `from_address`, `subject`, `recipients` and `message_id_header`
|
|
262
|
+
* (only once its raw copy is deleted). At least `rawRetentionDays`. Unset: kept. The conversation, tenant, status,
|
|
263
|
+
* verdicts and id stay. A later reply to a scrubbed message no longer matches it by Message-ID, so it starts a new
|
|
264
|
+
* conversation, and `mail.send` with that `inReplyTo` sends without reply headers.
|
|
265
|
+
*/
|
|
266
|
+
readonly metadataRetentionDays?: number
|
|
256
267
|
/** Days each stored attachment is kept, from receipt, before the storage sweep deletes it. Unset: no expiry. */
|
|
257
268
|
readonly attachmentRetentionDays?: number
|
|
258
269
|
/** Raw messages larger than this are marked `failed` and not handled. Default 40 MiB. */
|
|
@@ -268,7 +279,7 @@ export interface Inbound {
|
|
|
268
279
|
readonly jobs: readonly Handler<Record<string, unknown>, unknown>[]
|
|
269
280
|
/** Folder source only: receives every `.eml` file in the folder. Needs a Flow (a job or schedule run). */
|
|
270
281
|
scan(): Promise<InboundScanResult>
|
|
271
|
-
/** Deletes raw copies past their retention. Needs a Flow (a job or schedule run). */
|
|
282
|
+
/** Deletes raw copies past their retention and scrubs metadata past `metadataRetentionDays`. `limit` bounds each. Needs a Flow (a job or schedule run). */
|
|
272
283
|
sweep(options?: { readonly limit?: number }): Promise<InboundSweepResult>
|
|
273
284
|
/** The row of a received message, or `null`. Needs a Flow (a job, request or test context). */
|
|
274
285
|
get(id: string): Promise<InboundRecord | null>
|
|
@@ -336,6 +347,13 @@ export function createInbound(config: InboundConfig): Inbound {
|
|
|
336
347
|
const retentionDays = Math.floor(config.rawRetentionDays ?? DEFAULT_RETENTION_DAYS)
|
|
337
348
|
const maxRawBytes = config.maxRawBytes ?? DEFAULT_MAX_RAW_BYTES
|
|
338
349
|
if (!(retentionDays >= 0)) throw new Error('[pipework] inbound rawRetentionDays must be zero or more.')
|
|
350
|
+
const metadataDays = config.metadataRetentionDays
|
|
351
|
+
if (metadataDays !== undefined && !(Number.isInteger(metadataDays) && metadataDays >= 0)) {
|
|
352
|
+
throw new Error('[pipework] inbound metadataRetentionDays must be a whole number of zero or more.')
|
|
353
|
+
}
|
|
354
|
+
if (metadataDays !== undefined && metadataDays < retentionDays) {
|
|
355
|
+
throw new Error(`[pipework] inbound metadataRetentionDays (${String(metadataDays)}) must be at least rawRetentionDays (${String(retentionDays)}).`)
|
|
356
|
+
}
|
|
339
357
|
const attachmentDays = config.attachmentRetentionDays === undefined ? undefined : Math.floor(config.attachmentRetentionDays)
|
|
340
358
|
if (attachmentDays !== undefined && !(attachmentDays >= 1)) throw new Error('[pipework] inbound attachmentRetentionDays must be one or more.')
|
|
341
359
|
if (!(maxRawBytes > 0)) throw new Error('[pipework] inbound maxRawBytes must be positive.')
|
|
@@ -602,7 +620,20 @@ export function createInbound(config: InboundConfig): Inbound {
|
|
|
602
620
|
void recordFault('inbound-raw-delete', faultMessage(err), { inboundId: String(row['id']) })
|
|
603
621
|
}
|
|
604
622
|
}
|
|
605
|
-
|
|
623
|
+
let scrubbed = 0
|
|
624
|
+
if (metadataDays !== undefined) {
|
|
625
|
+
const rows = await db.execute(sql`
|
|
626
|
+
UPDATE pipework_email_inbound
|
|
627
|
+
SET from_address = NULL, subject = NULL, recipients = NULL, message_id_header = NULL, metadata_scrubbed_at = now()
|
|
628
|
+
WHERE id IN (
|
|
629
|
+
SELECT id FROM pipework_email_inbound
|
|
630
|
+
WHERE metadata_scrubbed_at IS NULL AND raw_deleted_at IS NOT NULL
|
|
631
|
+
AND received_at < now() - ${metadataDays} * interval '1 day'
|
|
632
|
+
ORDER BY received_at LIMIT ${limit})
|
|
633
|
+
RETURNING id`)
|
|
634
|
+
scrubbed = rows.length
|
|
635
|
+
}
|
|
636
|
+
return { deleted, failed, scrubbed }
|
|
606
637
|
},
|
|
607
638
|
|
|
608
639
|
async get(id) {
|
|
@@ -625,6 +656,7 @@ export function createInbound(config: InboundConfig): Inbound {
|
|
|
625
656
|
processedAt: date(row['processed_at']),
|
|
626
657
|
rawExpiresAt: toDate(row['raw_expires_at']),
|
|
627
658
|
rawDeletedAt: date(row['raw_deleted_at']),
|
|
659
|
+
metadataScrubbedAt: date(row['metadata_scrubbed_at']),
|
|
628
660
|
}
|
|
629
661
|
},
|
|
630
662
|
}
|
|
@@ -9,6 +9,7 @@ import { define } from '../../data/domain/index.js'
|
|
|
9
9
|
import { field } from '../../data/domain/field.js'
|
|
10
10
|
import { schema } from '../../data/schema/namespace.js'
|
|
11
11
|
import { index, unique } from '../../data/db/idx/index.js'
|
|
12
|
+
import { sql } from '../../data/db/sql.js'
|
|
12
13
|
|
|
13
14
|
export const EMAIL_TABLE_NAMES: ReadonlySet<string> = new Set([
|
|
14
15
|
'pipework_email_messages', 'pipework_email_suppressions', 'pipework_email_inbound', 'pipework_email_unsubscribes',
|
|
@@ -71,6 +72,8 @@ export const emailInbound = define('pipework_email_inbound', {
|
|
|
71
72
|
rawLocation: field.jsonb(schema.check.json()),
|
|
72
73
|
rawExpiresAt: field.timestamp(),
|
|
73
74
|
rawDeletedAt: field.timestamp().nullable(),
|
|
75
|
+
// Set when `sweep` nulls the address, subject, recipients and Message-ID after `metadataRetentionDays`.
|
|
76
|
+
metadataScrubbedAt: field.timestamp().nullable(),
|
|
74
77
|
status: field.enum(INBOUND_STATUSES).default('received'),
|
|
75
78
|
error: field.text().nullable(),
|
|
76
79
|
messageIdHeader: field.text().nullable(),
|
|
@@ -92,5 +95,6 @@ export const emailInbound = define('pipework_email_inbound', {
|
|
|
92
95
|
bySource: unique('pipework_email_inbound_source_id_key').on(t['sourceId']),
|
|
93
96
|
byMessageId: index('idx_pipework_email_inbound_message_id').on(t['messageIdHeader']),
|
|
94
97
|
byExpiry: index('idx_pipework_email_inbound_raw_expires').on(t['rawExpiresAt']),
|
|
98
|
+
unscrubbed: index('idx_pipework_email_inbound_unscrubbed').on(t['receivedAt']).where(sql.raw('metadata_scrubbed_at IS NULL')),
|
|
95
99
|
}),
|
|
96
100
|
})
|
package/src/infra/email/sns.ts
CHANGED
|
@@ -94,7 +94,7 @@ const processEvent = __flowStep({ ref: 'pipework/email:notification' }, async (e
|
|
|
94
94
|
} else {
|
|
95
95
|
getBaseLogger().info({ event: 'email bounce not suppressed', address, bounceType: r.bounceType }, 'transient bounce: address kept')
|
|
96
96
|
}
|
|
97
|
-
await auditEmail(r.kind, messageId, {
|
|
97
|
+
await auditEmail(r.kind, messageId, { bounceType: r.bounceType, suppressed: permanent, messageTraceId })
|
|
98
98
|
}
|
|
99
99
|
})
|
|
100
100
|
|
package/src/infra/outbox/core.ts
CHANGED
|
@@ -13,6 +13,9 @@ const LEASE_MS = 30_000
|
|
|
13
13
|
const BATCH = 50
|
|
14
14
|
const MAX_BATCHES = 20
|
|
15
15
|
const MAX_BACKOFF_MS = 3_600_000
|
|
16
|
+
const SWEEP_BATCH = 500
|
|
17
|
+
const DEFAULT_SWEEP_LIMIT = 10_000
|
|
18
|
+
const SWEEP_EVERY_MS = 3_600_000
|
|
16
19
|
|
|
17
20
|
export interface OutboxTotals {
|
|
18
21
|
delivered: number
|
|
@@ -61,6 +64,27 @@ export interface OutboxOptions<D> {
|
|
|
61
64
|
readonly onGivenUp: (row: Record<string, unknown>, attempt: number, error: string) => Promise<void>
|
|
62
65
|
}
|
|
63
66
|
|
|
67
|
+
/** Retention for an outbox's rows, in days. Unset: kept. */
|
|
68
|
+
export interface OutboxRetention {
|
|
69
|
+
/** Delivered rows (and their payloads) older than this are deleted. */
|
|
70
|
+
readonly deliveredRetentionDays: number
|
|
71
|
+
/** Given-up rows (`failed_at` set, never delivered) older than this are deleted. Unset: kept. */
|
|
72
|
+
readonly failedRetentionDays?: number
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Refuses a retention that is not whole days of zero or more, naming `who` and the option. */
|
|
76
|
+
export function checkRetention(who: string, config: { readonly deliveredRetentionDays?: number; readonly failedRetentionDays?: number }): OutboxRetention | undefined {
|
|
77
|
+
for (const name of ['deliveredRetentionDays', 'failedRetentionDays'] as const) {
|
|
78
|
+
const v = config[name]
|
|
79
|
+
if (v !== undefined && !(Number.isInteger(v) && v >= 0)) throw new Error(`[pipework] ${who}: ${name} must be an integer of zero or more, got ${String(v)}.`)
|
|
80
|
+
}
|
|
81
|
+
if (config.failedRetentionDays !== undefined && config.deliveredRetentionDays === undefined) {
|
|
82
|
+
throw new Error(`[pipework] ${who}: failedRetentionDays needs deliveredRetentionDays too.`)
|
|
83
|
+
}
|
|
84
|
+
if (config.deliveredRetentionDays === undefined) return undefined
|
|
85
|
+
return { deliveredRetentionDays: config.deliveredRetentionDays, ...(config.failedRetentionDays !== undefined ? { failedRetentionDays: config.failedRetentionDays } : {}) }
|
|
86
|
+
}
|
|
87
|
+
|
|
64
88
|
export interface Outbox {
|
|
65
89
|
/** Delivers what is due now, up to 20 batches; rows that fail wait for the next call. */
|
|
66
90
|
drain(): Promise<OutboxTotals>
|
|
@@ -70,6 +94,26 @@ export interface Outbox {
|
|
|
70
94
|
* `prev_attempts`, the attempts it had when given up.
|
|
71
95
|
*/
|
|
72
96
|
requeue(options: { sink?: string; failedSince?: Date }): Promise<Record<string, unknown>[]>
|
|
97
|
+
/**
|
|
98
|
+
* Deletes delivered rows whose `delivered_at` is older than `deliveredRetentionDays` and, with `failedRetentionDays`,
|
|
99
|
+
* given-up rows whose `failed_at` is older than that. A row still queued or being retried is never deleted.
|
|
100
|
+
* Deletes in batches of 500, at most `limit` rows per call (default 10 000).
|
|
101
|
+
*/
|
|
102
|
+
sweep(options: OutboxRetention & { limit?: number }): Promise<{ deleted: number }>
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** A function that runs `outbox.sweep` with `retention` at most once an hour per process, and never throws; `onError` gets a failure. No retention: does nothing. */
|
|
106
|
+
export function hourlySweep(outbox: Outbox, retention: OutboxRetention | undefined, onError: (err: unknown) => void): () => Promise<void> {
|
|
107
|
+
let last = 0
|
|
108
|
+
return async () => {
|
|
109
|
+
if (retention === undefined || Date.now() - last < SWEEP_EVERY_MS) return
|
|
110
|
+
last = Date.now()
|
|
111
|
+
try {
|
|
112
|
+
await outbox.sweep(retention)
|
|
113
|
+
} catch (err) {
|
|
114
|
+
onError(err)
|
|
115
|
+
}
|
|
116
|
+
}
|
|
73
117
|
}
|
|
74
118
|
|
|
75
119
|
function withTimeout(promise: Promise<void>, ms: number): Promise<void> {
|
|
@@ -136,7 +180,33 @@ export function createOutbox<D>(options: OutboxOptions<D>): Outbox {
|
|
|
136
180
|
}))
|
|
137
181
|
}
|
|
138
182
|
|
|
183
|
+
async function deleteBatch(predicate: ReturnType<typeof sql>, size: number): Promise<number> {
|
|
184
|
+
const rows = await db().execute(sql`
|
|
185
|
+
DELETE FROM ${table} WHERE id IN (SELECT id FROM ${table} WHERE ${predicate} LIMIT ${size}) RETURNING id`)
|
|
186
|
+
return rows.length
|
|
187
|
+
}
|
|
188
|
+
|
|
139
189
|
return {
|
|
190
|
+
async sweep(opts) {
|
|
191
|
+
const days = [['deliveredRetentionDays', opts.deliveredRetentionDays], ['failedRetentionDays', opts.failedRetentionDays], ['limit', opts.limit]] as const
|
|
192
|
+
for (const [name, v] of days) {
|
|
193
|
+
if (v !== undefined && !(Number.isInteger(v) && v >= (name === 'limit' ? 1 : 0))) {
|
|
194
|
+
throw new ValidationError({ [name]: ['must be a whole number' + (name === 'limit' ? ' of one or more' : ' of zero or more')] }, `Outbox sweep ${name} is invalid.`)
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
const delivered = sql`delivered_at IS NOT NULL AND delivered_at < clock_timestamp() - ${opts.deliveredRetentionDays} * interval '1 day'`
|
|
198
|
+
const failed = opts.failedRetentionDays === undefined ? null
|
|
199
|
+
: sql`delivered_at IS NULL AND failed_at IS NOT NULL AND failed_at < clock_timestamp() - ${opts.failedRetentionDays} * interval '1 day'`
|
|
200
|
+
const predicate = failed === null ? delivered : sql`((${delivered}) OR (${failed}))`
|
|
201
|
+
const limit = opts.limit ?? DEFAULT_SWEEP_LIMIT
|
|
202
|
+
let deleted = 0
|
|
203
|
+
while (deleted < limit) {
|
|
204
|
+
const n = await deleteBatch(predicate, Math.min(SWEEP_BATCH, limit - deleted))
|
|
205
|
+
deleted += n
|
|
206
|
+
if (n === 0) break
|
|
207
|
+
}
|
|
208
|
+
return { deleted }
|
|
209
|
+
},
|
|
140
210
|
async requeue(opts) {
|
|
141
211
|
if (opts.failedSince !== undefined && !(opts.failedSince instanceof Date && !Number.isNaN(opts.failedSince.getTime()))) {
|
|
142
212
|
throw new ValidationError({ failedSince: ['must be a valid Date'] }, 'Outbox failedSince must be a Date.')
|
|
@@ -17,7 +17,7 @@ import { recordBeat, beatsEnabled } from '../alerts/beat.js'
|
|
|
17
17
|
import type { Mail } from '../email/mail.js'
|
|
18
18
|
import { sanitizeHtml, htmlToText } from '../email/sanitize.js'
|
|
19
19
|
import type { ReceivedEmail } from '../email/inbound.js'
|
|
20
|
-
import { createOutbox, type Sink, type SinkDelivery } from '../outbox/core.js'
|
|
20
|
+
import { checkRetention, createOutbox, hourlySweep, type Sink, type SinkDelivery } from '../outbox/core.js'
|
|
21
21
|
import { TICKET_STATES } from './schema.js'
|
|
22
22
|
import { describeError } from '../../core/logging/error-chain.js'
|
|
23
23
|
|
|
@@ -87,18 +87,35 @@ export interface SupportConfig {
|
|
|
87
87
|
/**
|
|
88
88
|
* Who may join a ticket by thread. `'contact'` (default): only the ticket's contact; a message from anyone else in the
|
|
89
89
|
* thread opens its own ticket, and `receive` with a `ticketId` refuses a sender who is not the contact. `'conversation'`:
|
|
90
|
-
* a message threaded into the ticket's conversation, or sent to it with `ticketId`, joins it and reopens it
|
|
91
|
-
*
|
|
92
|
-
*
|
|
93
|
-
* a DMARC pass). A message `screen` separates still never joins.
|
|
90
|
+
* a message threaded into the ticket's conversation, or sent to it with `ticketId`, joins it and reopens it; the
|
|
91
|
+
* message keeps its own sender, and the ticket's contact and user do not change. Whether a sender other than the
|
|
92
|
+
* contact may join is `joinOthers`. A message `screen` separates never joins.
|
|
94
93
|
*/
|
|
95
94
|
readonly joinThread?: 'contact' | 'conversation'
|
|
95
|
+
/**
|
|
96
|
+
* With `joinThread: 'conversation'` (setting it with `'contact'` throws at create): whether a message from a sender
|
|
97
|
+
* other than the ticket's contact joins the ticket found. The contact's own message always joins. `'authentic'`
|
|
98
|
+
* (default): an email joins only when its DMARC verdict is `PASS`, or when it has no verdicts (the folder stand-in);
|
|
99
|
+
* `receive` (chat/api) joins, since the app vouched for the caller. `'any'`: joins. A function decides itself, with
|
|
100
|
+
* `email: null` for `receive`. A refused message is handled like one `screen` separates: it opens its own ticket,
|
|
101
|
+
* names no user, and its `receive` audit has `separated: true` and `reason: 'not-authentic'`. It never touches the
|
|
102
|
+
* found ticket (no drafts discarded, no reopen). `receive` with a refused `ticketId` throws ForbiddenError.
|
|
103
|
+
*/
|
|
104
|
+
readonly joinOthers?: 'authentic' | 'any' | ((who: { email: ReceivedEmail | null; ticket: Ticket }) => boolean)
|
|
96
105
|
/** Job type of `support.job`; default `'pipework.support.events'`. */
|
|
97
106
|
readonly jobType?: string
|
|
98
107
|
/** Delivery attempts before an event is given up on; default 10. */
|
|
99
108
|
readonly maxAttempts?: number
|
|
100
109
|
/** First retry delay, doubling per attempt up to one hour; default 5000. */
|
|
101
110
|
readonly backoffMs?: number
|
|
111
|
+
/**
|
|
112
|
+
* Days a delivered event row, with its payload (the ticket subject), is kept before the sweep deletes it. Unset: kept.
|
|
113
|
+
* `tick` (so `support.job`) runs the sweep at most once an hour per process; `sweep()` runs it on demand.
|
|
114
|
+
* The ticket audit rows carry ids only; they expire with the trace partitions (`auditRetentionDays`).
|
|
115
|
+
*/
|
|
116
|
+
readonly deliveredRetentionDays?: number
|
|
117
|
+
/** Days an event row that was given up on (never delivered) is kept. Needs `deliveredRetentionDays`. Unset: kept. */
|
|
118
|
+
readonly failedRetentionDays?: number
|
|
102
119
|
}
|
|
103
120
|
|
|
104
121
|
export interface Ticket {
|
|
@@ -256,6 +273,8 @@ export interface Support extends WithFor<TicketApi> {
|
|
|
256
273
|
readonly job: Handler<Record<string, unknown>, unknown>
|
|
257
274
|
/** What one run of `job` does; callable from a test. Needs a Flow. */
|
|
258
275
|
tick(): Promise<{ delivered: number; retried: number; failed: number }>
|
|
276
|
+
/** Deletes event rows past `deliveredRetentionDays` / `failedRetentionDays` now, up to `limit` (default 10 000); `deleted: 0` when neither is set. Needs a Flow. */
|
|
277
|
+
sweep(options?: { limit?: number }): Promise<{ deleted: number }>
|
|
259
278
|
/**
|
|
260
279
|
* Puts the event deliveries that were given up (and never delivered) back on the queue, for every tenant, optionally
|
|
261
280
|
* only those of `sink` and/or that failed at or after `failedSince`; the next `tick` sends them. Delivery ids stay the
|
|
@@ -358,6 +377,14 @@ export function createSupport(config: SupportConfig): Support {
|
|
|
358
377
|
if ((joinThread as string) !== 'contact' && (joinThread as string) !== 'conversation') {
|
|
359
378
|
throw new Error(`[pipework] support.create: joinThread must be 'contact' or 'conversation', got ${JSON.stringify(joinThread)}.`)
|
|
360
379
|
}
|
|
380
|
+
const joinOthers = config.joinOthers ?? 'authentic'
|
|
381
|
+
if (config.joinOthers !== undefined && joinThread !== 'conversation') {
|
|
382
|
+
throw new Error(`[pipework] support.create: joinOthers applies only with joinThread: 'conversation'; remove it or set joinThread.`)
|
|
383
|
+
}
|
|
384
|
+
if (joinOthers !== 'authentic' && joinOthers !== 'any' && typeof joinOthers !== 'function') {
|
|
385
|
+
throw new Error(`[pipework] support.create: joinOthers must be 'authentic', 'any' or a function, got ${JSON.stringify(joinOthers)}.`)
|
|
386
|
+
}
|
|
387
|
+
const retention = checkRetention('support.create', config)
|
|
361
388
|
const jobType = config.jobType ?? 'pipework.support.events'
|
|
362
389
|
const replyTemplate = config.replyTemplate ?? 'pipework.support.reply'
|
|
363
390
|
const distinctApprover = config.distinctApprover ?? true
|
|
@@ -503,12 +530,21 @@ export function createSupport(config: SupportConfig): Support {
|
|
|
503
530
|
if (seen[0] !== undefined) return { status: 'duplicate', ticketId: seen[0]['ticket_id'] as string, created: false }
|
|
504
531
|
}
|
|
505
532
|
|
|
533
|
+
let separate = m.separate === true
|
|
534
|
+
let reason: string | null = null
|
|
535
|
+
const mayJoin = (t: Ticket): boolean => {
|
|
536
|
+
if (t.contactEmail === m.contactEmail || joinOthers === 'any') return true
|
|
537
|
+
if (typeof joinOthers === 'function') return joinOthers({ email: m.email ?? null, ticket: t })
|
|
538
|
+
const v = m.email?.verdicts
|
|
539
|
+
return m.email == null || v == null || v.dmarc === 'PASS'
|
|
540
|
+
}
|
|
506
541
|
let ticket: Ticket | null = null
|
|
507
542
|
if (m.ticketId !== null) {
|
|
508
543
|
ticket = await loadTicket(tx, tenantId, m.ticketId, true)
|
|
509
544
|
if (joinThread === 'contact' && ticket.contactEmail !== m.contactEmail) {
|
|
510
545
|
throw new ForbiddenError('The sender is not the ticket\'s contact.', { ticketId: m.ticketId })
|
|
511
546
|
}
|
|
547
|
+
if (!mayJoin(ticket)) throw new ForbiddenError('The sender may not join the ticket (joinOthers).', { ticketId: m.ticketId })
|
|
512
548
|
} else if (m.conversationId !== null && m.separate !== true) {
|
|
513
549
|
const contactOnly = joinThread === 'contact' ? sql`AND t.contact_email = ${m.contactEmail}` : sql``
|
|
514
550
|
const found = await tx.execute(sql`
|
|
@@ -518,12 +554,19 @@ export function createSupport(config: SupportConfig): Support {
|
|
|
518
554
|
SELECT 1 FROM support_message sm JOIN pipework_email_inbound i ON i.id = sm.inbound_email_id
|
|
519
555
|
WHERE sm.ticket_id = t.id AND i.conversation_id = ${m.conversationId}::uuid))
|
|
520
556
|
ORDER BY (t.conversation_id = ${m.conversationId}::uuid) DESC NULLS LAST, t.created_at DESC, t.id LIMIT 1`)
|
|
521
|
-
if (found[0] !== undefined)
|
|
557
|
+
if (found[0] !== undefined) {
|
|
558
|
+
const t = await loadTicket(tx, tenantId, found[0]['id'] as string, true)
|
|
559
|
+
if (mayJoin(t)) ticket = t
|
|
560
|
+
else {
|
|
561
|
+
separate = true
|
|
562
|
+
reason = 'not-authentic'
|
|
563
|
+
}
|
|
564
|
+
}
|
|
522
565
|
}
|
|
523
566
|
|
|
524
567
|
const resolve = async (): Promise<string | null> => {
|
|
525
568
|
if (m.userId !== null) return m.userId
|
|
526
|
-
if (
|
|
569
|
+
if (separate) return null
|
|
527
570
|
const found = await config.resolveContact?.({ address: m.contactEmail, name: m.contactName, tenantId, email: m.email ?? null }, tx)
|
|
528
571
|
return found?.userId ?? null
|
|
529
572
|
}
|
|
@@ -534,7 +577,7 @@ export function createSupport(config: SupportConfig): Support {
|
|
|
534
577
|
if (ticket === null) {
|
|
535
578
|
const userId = await resolve()
|
|
536
579
|
let conversationId = m.conversationId
|
|
537
|
-
if (
|
|
580
|
+
if (separate) conversationId = null
|
|
538
581
|
if (conversationId !== null) {
|
|
539
582
|
const held = await tx.execute(sql`SELECT 1 FROM support_ticket WHERE tenant_id = ${tenantId}::uuid AND conversation_id = ${conversationId}::uuid`)
|
|
540
583
|
if (held.length > 0) conversationId = null
|
|
@@ -545,7 +588,7 @@ export function createSupport(config: SupportConfig): Support {
|
|
|
545
588
|
RETURNING *`)
|
|
546
589
|
ticket = toTicket(rows[0]!)
|
|
547
590
|
created = true
|
|
548
|
-
await audit(flow, tx, tenantId, 'create', ticket.id, {
|
|
591
|
+
await audit(flow, tx, tenantId, 'create', ticket.id, { channel: m.channel, userId: ticket.userId })
|
|
549
592
|
} else {
|
|
550
593
|
await discardLive(flow, tx, ticket, 'customer-replied')
|
|
551
594
|
if (ticket.state !== 'open') ticket = await moveTo(flow, tx, ticket, 'open', true)
|
|
@@ -562,7 +605,7 @@ export function createSupport(config: SupportConfig): Support {
|
|
|
562
605
|
INSERT INTO support_message (tenant_id, ticket_id, kind, channel, text, html, from_address, inbound_email_id, attachments)
|
|
563
606
|
VALUES (${tenantId}::uuid, ${ticket.id}::uuid, 'inbound', ${m.channel}, ${text}, ${html}, ${m.fromAddress},
|
|
564
607
|
${m.inboundEmailId}::uuid, ${m.attachments === null ? null : JSON.stringify(m.attachments)}::jsonb)`)
|
|
565
|
-
await audit(flow, tx, tenantId, 'receive', ticket.id, { channel: m.channel, emailId: m.inboundEmailId, created, separated:
|
|
608
|
+
await audit(flow, tx, tenantId, 'receive', ticket.id, { channel: m.channel, emailId: m.inboundEmailId, created, separated: separate, ...(reason === null ? {} : { reason }), ...(joinedBy === null ? {} : { joinedBy }), ...(userIdSet === null ? {} : { userIdSet }) })
|
|
566
609
|
await emit(tx, created ? 'ticket.created' : 'ticket.customer-replied', ticket)
|
|
567
610
|
return { status: 'received', ticketId: ticket.id, created }
|
|
568
611
|
}
|
|
@@ -802,7 +845,15 @@ export function createSupport(config: SupportConfig): Support {
|
|
|
802
845
|
await audit(flow, null, payload.tenantId, verb, row['ticket_id'] as string, { sink: row['sink'], kind: row['kind'], ...extra })
|
|
803
846
|
}
|
|
804
847
|
|
|
805
|
-
const
|
|
848
|
+
const sweepIfDue = hourlySweep(outbox, retention, (err) => {
|
|
849
|
+
getBaseLogger().error({ err }, 'support event sweep failed')
|
|
850
|
+
void recordFault('support-event-sweep', faultMessage(err), {})
|
|
851
|
+
})
|
|
852
|
+
const drainStep = __flowStep({ ref: 'pipework/support:tick' }, async () => {
|
|
853
|
+
const totals = await outbox.drain()
|
|
854
|
+
await sweepIfDue()
|
|
855
|
+
return totals
|
|
856
|
+
})
|
|
806
857
|
// A beat for `alerts.source.loopStalled`, written only where the blueprint has `alerts`.
|
|
807
858
|
const beat = async (outcome: { ok: boolean; error?: string }): Promise<void> => {
|
|
808
859
|
const ctx = getContext()
|
|
@@ -855,6 +906,10 @@ export function createSupport(config: SupportConfig): Support {
|
|
|
855
906
|
},
|
|
856
907
|
job: job as unknown as Handler<Record<string, unknown>, unknown>,
|
|
857
908
|
tick: () => tickStep(),
|
|
909
|
+
async sweep(options = {}) {
|
|
910
|
+
requireContext()
|
|
911
|
+
return retention === undefined ? { deleted: 0 } : outbox.sweep({ ...retention, ...(options.limit !== undefined ? { limit: options.limit } : {}) })
|
|
912
|
+
},
|
|
858
913
|
async requeueFailed(options) {
|
|
859
914
|
requireContext()
|
|
860
915
|
nonEmpty('by', options.by)
|