pipework 0.275.0 → 0.277.0

package/src/infra/alerts/schema.ts

@@ -36,6 +36,8 @@ export const alerts = define('pipework_alerts', {
   issueUrl: field.text().nullable(),
   traceIds: field.custom(schema.col.text('trace_ids').array(), schema.check.array(schema.check.string())).default('{}'),
   resendAfterMs: field.integer().nullable(),
+  resendOnNewRelease: field.boolean().nullable(),
+  lastSentRelease: field.text().nullable(),
   resolveAfterQuietMs: field.integer().nullable(),
   updatedAt: field.timestamp().default('now'),
 }, {
@@ -44,6 +46,12 @@ export const alerts = define('pipework_alerts', {
   }),
 })
 
+// One row per release a process running alerts has recorded, with when it was first seen; `resendOnNewRelease` compares these.
+export const alertReleases = define('pipework_alert_releases', {
+  release: field.text().primaryKey(),
+  firstSeenAt: field.timestamp().default('now'),
+})
+
 // One row per raise, kept for `occurrenceRetentionMs`.
 export const alertOccurrences = define('pipework_alert_occurrences', {
   id: field.uuid().defaultRandom().primaryKey(),

package/src/request/context/run-in-transaction.ts

@@ -48,14 +48,14 @@ export async function executeInTransaction<T>(
 
       await propagateLocals(tx, ctx, tenant)
 
-      const disarm = await armCancellation(conn, tx, signal)
+      const cancellation = await armCancellation(conn, tx, signal)
       try {
         const result = await runInContext(childCtx, run)
         // Idle at the abort, nothing failed: throwing here rolls back instead of committing.
         signal?.throwIfAborted()
         return result
       } finally {
-        disarm()
+        await cancellation.settle()
       }
     })
   }
@@ -90,10 +90,15 @@ export async function executeInTransaction<T>(
       conn.drizzle.transaction(async (tx) => {
         childCtx.transactions.set(dbName, bindTransaction(ctx, dbName, tx, signal))
         await propagateLocals(tx, ctx, tenant)
-        disarms.push(await armCancellation(conn, tx, signal))
+        const cancellation = await armCancellation(conn, tx, signal)
+        disarms.push(cancellation.disarm)
         readyDatabases.add(dbName)
         signalReady()
-        await donePromise
+        try {
+          await donePromise
+        } finally {
+          await cancellation.settle()
+        }
       }),
     )
   }
@@ -142,34 +147,38 @@ export async function executeInTransaction<T>(
  * Arms server-side cancellation for one transaction: on abort, a separate
  * pooled connection issues `pg_cancel_backend` against the backend this
  * transaction holds, so an in-flight query fails with `query_canceled` and the
- * transaction unwinds. Returns a disarm function — always call it, or the
- * listener outlives the transaction and cancels a backend the pool has since
- * handed to someone else.
+ * transaction unwinds. Always call `settle()` before the transaction ends: it
+ * removes the listener and waits for a cancel already sent to be delivered, so
+ * none reaches a backend the pool has since handed to someone else. `disarm()`
+ * only removes the listener.
  */
 async function armCancellation(
   conn: ManagedConnection,
   tx: DB,
   signal: AbortSignal | undefined,
-): Promise<() => void> {
-  if (signal === undefined) return () => {}
+): Promise<{ disarm: () => void; settle: () => Promise<void> }> {
+  const none = { disarm: () => {}, settle: async () => {} }
+  if (signal === undefined) return none
 
   const [row] = await tx.execute(sql`SELECT pg_backend_pid() AS pid`)
   const pid = Number((row as { pid?: unknown } | undefined)?.pid)
-  if (!Number.isInteger(pid)) return () => {}
+  if (!Number.isInteger(pid)) return none
 
+  let pending: Promise<void> | undefined
   const cancel = (): void => {
     // Best effort: the backend may already have finished, and a failed cancel
     // must not replace the abort the caller is about to see.
-    void conn.drizzle.execute(sql`SELECT pg_cancel_backend(${pid})`).catch(() => {})
+    pending = (conn.cancelBackend?.(pid) ?? conn.drizzle.execute(sql`SELECT pg_cancel_backend(${pid})`)).then(() => {}, () => {})
   }
-
-  if (signal.aborted) {
-    cancel()
-    return () => {}
+  const disarm = (): void => { signal.removeEventListener('abort', cancel) }
+  const settle = async (): Promise<void> => {
+    disarm()
+    await pending
   }
 
-  signal.addEventListener('abort', cancel, { once: true })
-  return () => { signal.removeEventListener('abort', cancel) }
+  if (signal.aborted) cancel()
+  else signal.addEventListener('abort', cancel, { once: true })
+  return { disarm, settle }
 }
 
 /**

package/src/request/http/aborted.ts

@@ -13,3 +13,22 @@ export class RequestAbortedError extends PipeworkError {
     super(`[pipework] The client closed the connection before the response was sent (${method} ${route}).`, { method, route })
   }
 }
+
+/**
+ * Whether `error` is something the abort itself caused: anywhere in its `cause`
+ * chain, the signal's reason, a `RequestAbortedError`, an `AbortError`, or a
+ * Postgres `query_canceled` (57014) / `in_failed_sql_transaction` (25P02).
+ * Any other error after an abort is a real failure.
+ */
+export function isAbortCaused(error: unknown, signal: AbortSignal): boolean {
+  const seen = new Set<unknown>()
+  let current: unknown = error
+  while (current !== null && current !== undefined && !seen.has(current)) {
+    seen.add(current)
+    if (current === signal.reason || current instanceof RequestAbortedError) return true
+    const { name, code, cause } = current as { name?: unknown; code?: unknown; cause?: unknown }
+    if (name === 'AbortError' || code === '57014' || code === '25P02') return true
+    current = cause
+  }
+  return false
+}

package/src/request/http/namespace.ts

@@ -3,5 +3,5 @@ import { detectMediaType, resolveMediaType, SNIFF_BYTES } from './sniff.js'
 
 /** HTTP namespace: `http.file()` builds a file response; `http.detectMediaType()` names bytes the way uploads do; the server itself is managed by manifold.start().
  *
- * A client that disconnects before its response is finished aborts the request: the in-flight statement is cancelled, the transaction rolled back and the connection released. The context's `signal` (`flow.require().signal`) aborts with a `RequestAbortedError` (499, `REQUEST_ABORTED`); pass it to the handler's own outbound calls (`fetch(url, { signal })`, a `quota.call` send). The aborted request is not an internal error: `onInternalError` is not called and nothing is sent. A route opts out with `.route(method, path, { cancelOnDisconnect: false })`, e.g. a webhook receiver whose sender abandons slow deliveries and does not redeliver: its signal never aborts and the transaction commits after the socket closes. */
+ * A client that disconnects before its response is finished aborts the request: the in-flight statement is cancelled, the transaction rolled back and the connection released. The context's `signal` (`flow.require().signal`) aborts with a `RequestAbortedError` (499, `REQUEST_ABORTED`); pass it to the handler's own outbound calls (`fetch(url, { signal })`, a `quota.call` send). Nothing is sent, whether the handler then throws or completes (an injected call rejects with `AbortError`). What the abort caused (its reason, a cancelled statement) is not an internal error and `onInternalError` is not called; any other error thrown after the abort still reaches it. A route opts out with `.route(method, path, { cancelOnDisconnect: false })`, e.g. a webhook receiver whose sender abandons slow deliveries and does not redeliver: its signal never aborts and the transaction commits after the socket closes. */
 export const http = { file, detectMediaType, resolveMediaType, SNIFF_BYTES } as const

package/src/request/http/server.ts

@@ -13,7 +13,7 @@ import { createRequestContext } from '../context/index.js'
 import { executeInTransaction } from '../context/run-in-transaction.js'
 import { resolveAndExecute, parseRouteInputs, resolveHandlerDatabases, buildCheckpoint, type ResolveOptions } from '../di/resolve.js'
 import { createRateLimitClasses } from './rate-limit-classes.js'
-import { RequestAbortedError } from './aborted.js'
+import { RequestAbortedError, isAbortCaused } from './aborted.js'
 import { buildContext, type ContextOptions } from './middleware.js'
 import { closeTrace } from '../../trace/entry/close-trace.js'
 import { recordErrorStep } from './error-step.js'
@@ -39,6 +39,9 @@ import { withTenantSetting } from '../../auth/tenant/in-tenant.js'
 import type { Resource } from '../fixture/types.js'
 import { resourceToHandlers } from '../fixture/to-handlers.js'
 
+/** The errors an aborted, injected request's raw response was destroyed with. */
+const settledAborts = new WeakSet<object>()
+
 /** Fastify server wrapper with route registration, handler auto-wiring, and inject() for testing. */
 export interface PipeworkServer {
   route(method: HttpMethod, path: string, handler: Handler<Record<string, unknown>, unknown>): void
@@ -169,6 +172,8 @@ export function createServer(instance: Manifold, config: ServerConfig = {}): Pip
   }
 
   app.setErrorHandler(async (error, req, reply) => {
+    // The error an aborted request's raw response was destroyed with: not a fault, nothing to send.
+    if (typeof error === 'object' && error !== null && settledAborts.has(error)) return reply
     const status = (error as { statusCode?: unknown }).statusCode
     const open = requestFlows.get(req)
     const routeOf = (): string => (req.routeOptions as { url?: string } | undefined)?.url ?? req.url.split('?')[0]!
@@ -458,6 +463,20 @@ export function createServer(instance: Manifold, config: ServerConfig = {}): Pip
           }
         }
       }
+      // An aborted request is never answered: the raw response is destroyed, so an
+      // injected caller's promise rejects with AbortError and a real socket stays closed.
+      const settleAborted = (): FastifyReply => {
+        const injected = (req.raw as { _lightMyRequest?: unknown })._lightMyRequest !== undefined
+        if (reply.raw.destroyed) return reply
+        if (!injected) {
+          reply.raw.destroy()
+          return reply
+        }
+        const abortError = new DOMException('The request was aborted', 'AbortError')
+        settledAborts.add(abortError)
+        reply.raw.destroy(abortError)
+        return reply
+      }
       // Ends the upload stream and closes the connection if the body was left unread.
       const finishUpload = (failed: boolean): void => {
         if (settleUpload(uploaded?.stream) || (failed && upload !== null)) void reply.header('connection', 'close')
@@ -536,20 +555,29 @@ export function createServer(instance: Manifold, config: ServerConfig = {}): Pip
           result = await runInContext(ctx, () => resolveAndExecute(handler, body, req.query, req.params, httpOptions))
         }
 
+        if (abort.signal.aborted) {
+          finishUpload(true)
+          recordErrorStep(ctx, { method, route: path, status: 499, code: 'REQUEST_ABORTED', error: abort.signal.reason })
+          void closeTrace(instance, ctx.trace, 'error')
+          req.log.info({ route: `${method} ${path}`, requestId: String(req.id) }, 'request aborted by client')
+          return settleAborted()
+        }
         finishUpload(false)
         void closeTrace(instance, ctx.trace, 'closed')
         return await sendResult(reply, req, result, handler.meta.file ?? null, `${method} ${path}`)
       } catch (error) {
         finishUpload(true)
-        // The client is gone, whatever the error was (a cancelled statement, the abort itself):
-        // not a fault, nothing to send.
-        if (abort.signal.aborted) {
+        // The client is gone, so nothing is sent. What the abort caused (the abort itself, a
+        // cancelled statement) is not a fault; any other error is classified and reported as usual.
+        const aborted = abort.signal.aborted
+        if (aborted && isAbortCaused(error, abort.signal)) {
           const reason: unknown = abort.signal.reason
           recordErrorStep(ctx, { method, route: path, status: 499, code: 'REQUEST_ABORTED', error: reason })
           if (ctx !== undefined) void closeTrace(instance, ctx.trace, 'error')
           req.log.info({ route: `${method} ${path}`, requestId: String(req.id) }, 'request aborted by client')
-          return reply
+          return settleAborted()
         }
+        const send = (status: number, body: unknown): FastifyReply => (aborted ? settleAborted() : sendError(reply, status, body))
         // Records the error step, runs `refused` (the route's onRefused hook) while the
         // trace is still open, then closes the trace.
         const failWith = async (status: number, code: string, refused?: () => Promise<void>): Promise<void> => {
@@ -565,7 +593,7 @@ export function createServer(instance: Manifold, config: ServerConfig = {}): Pip
           await failWith(500, 'FILE_RESPONSE_INVALID')
           req.log.error({ route: `${method} ${path}`, requestId: String(req.id), err: error }, 'File response refused')
           await reportInternalError(req, error, handler)
-          return sendError(reply, 500, { error: { code: 'FILE_RESPONSE_INVALID', message: error.message } })
+          return send(500, { error: { code: 'FILE_RESPONSE_INVALID', message: error.message } })
         }
 
         if (isPipeworkError(error)) {
@@ -581,7 +609,7 @@ export function createServer(instance: Manifold, config: ServerConfig = {}): Pip
           const retryAfter = response.error.retryAfterSeconds
           if (retryAfter !== undefined) void reply.header('Retry-After', String(retryAfter))
           if (error.httpStatus >= 500) await reportInternalError(req, error, handler)
-          return sendError(reply, error.httpStatus, response)
+          return send(error.httpStatus, response)
         }
 
         // A malformed multipart body surfaces as a Fastify client error (4xx).
@@ -589,13 +617,13 @@ export function createServer(instance: Manifold, config: ServerConfig = {}): Pip
         if (typeof clientStatus === 'number' && clientStatus >= 400 && clientStatus < 500) {
           const response = buildClientErrorResponse(clientStatus, error)
           await failWith(clientStatus, response.error.code, () => runRefusedHook(req, handler.meta, ctx, clientStatus, response))
-          return sendError(reply, clientStatus, response)
+          return send(clientStatus, response)
         }
 
         await failWith(500, 'INTERNAL_ERROR')
         req.log.error(error, 'Unhandled error in route handler')
         await reportInternalError(req, error, handler)
-        return sendError(reply, 500, buildInternalErrorResponse(error, config.exposeErrorDetails))
+        return send(500, buildInternalErrorResponse(error, config.exposeErrorDetails))
       }
     }
 

package/src/test/plugin.ts

@@ -73,6 +73,7 @@ export async function beginTestIsolation(instance: Manifold, task?: WatchedTask)
           '  Fix: call connectionIds() on a pool opened with instance.pool.getOrCreate() or tap(), outside test isolation.',
         ))
       },
+      async cancelBackend() { /* the one pinned test connection is never cancelled */ },
       async close() { /* test harness manages lifecycle */ },
     })
   }