tapimo 0.9.0 → 0.10.1

package/src/db/tables/webhookQueueEvents.ts

@@ -0,0 +1,285 @@
+import { type ExpressionBuilder, type Generated, type Insertable, type Selectable, sql } from 'kysely' // prettier-ignore
+
+import type * as Webhooks from '../../internal/Webhooks.js'
+import type * as Db from '../Db.js'
+import type * as db_Schema from '../Schema.js'
+
+/** Columns of the `webhook_queue_events` table, derived from `Schema.WebhookQueueEvent`. */
+export type Table = Omit<
+  db_Schema.WebhookQueueEvent,
+  'attemptCount' | 'attemptingAt' | 'nextAttemptAt' | 'status'
+> & {
+  /** Delivery attempts claimed so far; database default 0. */
+  attemptCount: Generated<number>
+  /** ISO timestamp of the current claim; database default null. */
+  attemptingAt: Generated<string | null>
+  /** ISO retry due time; database default null (due immediately). */
+  nextAttemptAt: Generated<string | null>
+  /** Delivery-obligation lifecycle status; database default `pending`. */
+  status: Generated<db_Schema.WebhookQueueEvent['status']>
+}
+
+/** A stored webhook Queue event. */
+export type Record = Selectable<Table>
+
+/** A terminal delivery-obligation outcome. */
+export type TerminalStatus = Exclude<db_Schema.WebhookQueueEvent['status'], 'pending'>
+
+/**
+ * Reads one pending envelope. Terminal rows read as absent so no consumer
+ * can redeliver a completed obligation or disturb its dedupe row.
+ */
+export async function get(
+  db: Db.Db,
+  reference: Webhooks.QueueReference,
+): Promise<Webhooks.QueueEvent | null> {
+  return (
+    (await db.kysely
+      .selectFrom('webhook_queue_events')
+      .selectAll()
+      .where('eventId', '=', reference.eventId)
+      .where('status', '=', 'pending')
+      .where('subscriptionId', '=', reference.subscriptionId)
+      .executeTakeFirst()) ?? null
+  )
+}
+
+/**
+ * Stages envelopes as fresh obligations. Pending conflicts keep their first
+ * envelope and backoff position; terminal conflicts inside the dedupe window
+ * suppress the replay; terminal conflicts past it are replaced, so an expired
+ * dedupe row pruning has not yet reclaimed never blocks a replay.
+ */
+export async function ensure(db: Db.Db, rows: readonly Insertable<Table>[]): Promise<number> {
+  if (rows.length === 0) return 0
+  const result = await db.kysely
+    .insertInto('webhook_queue_events')
+    .values([...rows])
+    .onConflict((oc) =>
+      oc
+        .columns(['subscriptionId', 'eventId'])
+        .doUpdateSet((eb) => ({
+          attemptCount: 0,
+          attemptingAt: null,
+          createdAt: eb.ref('excluded.createdAt'),
+          envelope: eb.ref('excluded.envelope'),
+          expiresAt: eb.ref('excluded.expiresAt'),
+          nextAttemptAt: eb.ref('excluded.nextAttemptAt'),
+          status: 'pending',
+        }))
+        .where((eb) =>
+          eb.and([
+            eb('webhook_queue_events.status', '!=', 'pending'),
+            eb('webhook_queue_events.expiresAt', '<=', eb.ref('excluded.createdAt')),
+          ]),
+        ),
+    )
+    .executeTakeFirst()
+  // The command tag counts inserted rows plus replaced expired rows; live
+  // conflicts fall out of the conditional update and are not counted.
+  return Number(result.numInsertedOrUpdatedRows ?? 0n)
+}
+
+/** Due, unclaimed pending work: shared by {@link claim} and {@link due}. */
+function whereClaimable(
+  eb: ExpressionBuilder<Db.Database, 'webhook_queue_events'>,
+  options: { now: string; staleBefore: string },
+) {
+  return eb.and([
+    eb('status', '=', 'pending'),
+    eb.or([eb('nextAttemptAt', 'is', null), eb('nextAttemptAt', '<=', options.now)]),
+    eb.or([eb('attemptingAt', 'is', null), eb('attemptingAt', '<=', options.staleBefore)]),
+  ])
+}
+
+/** Claims one due pending obligation with a single atomic update. */
+export async function claim(
+  db: Db.Db,
+  reference: Webhooks.QueueReference,
+  options: claim.Options,
+): Promise<claim.Result> {
+  const claimed = await db.kysely
+    .updateTable('webhook_queue_events')
+    .set((eb) => ({ attemptCount: eb('attemptCount', '+', 1), attemptingAt: options.now }))
+    .where('subscriptionId', '=', reference.subscriptionId)
+    .where('eventId', '=', reference.eventId)
+    .where((eb) => whereClaimable(eb, options))
+    .returningAll()
+    .executeTakeFirst()
+  if (claimed) return { record: claimed, type: 'claimed' }
+
+  const existing = await db.kysely
+    .selectFrom('webhook_queue_events')
+    .select(['attemptingAt', 'nextAttemptAt', 'status'])
+    .where('subscriptionId', '=', reference.subscriptionId)
+    .where('eventId', '=', reference.eventId)
+    .executeTakeFirst()
+  if (!existing) return { type: 'missing' }
+  if (existing.status !== 'pending') return { status: existing.status, type: 'terminal' }
+  if (existing.attemptingAt !== null && existing.attemptingAt > options.staleBefore)
+    return { type: 'attempting' }
+  return { nextAttemptAt: existing.nextAttemptAt, type: 'scheduled' }
+}
+
+export declare namespace claim {
+  /** Claim clock bounds. */
+  type Options = {
+    /** Current ISO timestamp; becomes the claim marker. */
+    now: string
+    /** ISO cutoff below which an existing claim counts as stale. */
+    staleBefore: string
+  }
+
+  /** Atomic claim outcome. */
+  type Result =
+    | { type: 'attempting' }
+    | { record: Record; type: 'claimed' }
+    | { type: 'missing' }
+    | { nextAttemptAt: string | null; type: 'scheduled' }
+    | { status: TerminalStatus; type: 'terminal' }
+}
+
+/**
+ * Marks a claimed obligation terminal, starting its dedupe retention window;
+ * pending rows never expire, so this stamp is the only expiry that matters.
+ * The `attemptingAt` predicate fences stale claimants.
+ */
+export async function complete(
+  db: Db.Db,
+  reference: Webhooks.QueueReference,
+  options: complete.Options,
+): Promise<boolean> {
+  const result = await db.kysely
+    .updateTable('webhook_queue_events')
+    .set({ attemptingAt: null, expiresAt: options.expiresAt, status: options.status })
+    .where('subscriptionId', '=', reference.subscriptionId)
+    .where('eventId', '=', reference.eventId)
+    .where('status', '=', 'pending')
+    .where('attemptingAt', '=', options.claimedAt)
+    .executeTakeFirst()
+  return result.numUpdatedRows === 1n
+}
+
+export declare namespace complete {
+  /** Fenced terminal transition. */
+  type Options = {
+    /** Claim marker returned by {@link claim}; fences stale claimants. */
+    claimedAt: string
+    /** Replacement ISO retention deadline for the terminal dedupe row. */
+    expiresAt: string
+    /** Terminal outcome. */
+    status: TerminalStatus
+  }
+}
+
+/** Releases a claim and schedules the next attempt; fenced like {@link complete}. */
+export async function schedule(
+  db: Db.Db,
+  reference: Webhooks.QueueReference,
+  options: schedule.Options,
+): Promise<boolean> {
+  const result = await db.kysely
+    .updateTable('webhook_queue_events')
+    .set({ attemptingAt: null, nextAttemptAt: options.nextAttemptAt })
+    .where('subscriptionId', '=', reference.subscriptionId)
+    .where('eventId', '=', reference.eventId)
+    .where('status', '=', 'pending')
+    .where('attemptingAt', '=', options.claimedAt)
+    .executeTakeFirst()
+  return result.numUpdatedRows === 1n
+}
+
+export declare namespace schedule {
+  /** Fenced retry scheduling. */
+  type Options = {
+    /** Claim marker returned by {@link claim}; fences stale claimants. */
+    claimedAt: string
+    /** ISO due time of the next attempt. */
+    nextAttemptAt: string
+  }
+}
+
+/** Lists due pending references for the sweeper, most overdue first. */
+export async function due(db: Db.Db, options: due.Options): Promise<Webhooks.QueueReference[]> {
+  return (
+    db.kysely
+      .selectFrom('webhook_queue_events')
+      .select(['eventId', 'subscriptionId'])
+      .where((eb) => whereClaimable(eb, options))
+      // Matches the pending partial index order so the scan needs no sort.
+      .orderBy(sql`next_attempt_at nulls first`)
+      .limit(options.limit)
+      .execute()
+  )
+}
+
+export declare namespace due {
+  /** Sweep bounds. */
+  type Options = {
+    /** Maximum references to return. */
+    limit: number
+    /** Current ISO timestamp. */
+    now: string
+    /** ISO cutoff below which an existing claim counts as stale. */
+    staleBefore: string
+  }
+}
+
+/** Returns the pending-obligation snapshot, split by due-and-unclaimed work. */
+export async function pending(db: Db.Db, options: pending.Options): Promise<pending.Result> {
+  const row = await db.kysely
+    .selectFrom('webhook_queue_events')
+    .select((eb) => [
+      eb.fn.countAll<number>().as('count'),
+      eb.fn.min('createdAt').as('oldestAt'),
+      eb.fn
+        .countAll<number>()
+        .filterWhere((fb) => whereClaimable(fb, options))
+        .as('dueCount'),
+      eb.fn
+        // Null next attempts (migration-backfilled rows) are due immediately;
+        // coalesce to staging time so they age instead of vanishing from MIN.
+        .min(eb.fn.coalesce('nextAttemptAt', 'createdAt'))
+        .filterWhere((fb) => whereClaimable(fb, options))
+        .as('oldestDueAt'),
+    ])
+    .where('status', '=', 'pending')
+    .executeTakeFirstOrThrow()
+  return {
+    count: Number(row.count),
+    dueCount: Number(row.dueCount),
+    oldestAt: row.oldestAt,
+    oldestDueAt: row.oldestDueAt,
+  }
+}
+
+export declare namespace pending {
+  /** Snapshot clock bounds; the read-only subset of {@link claim.Options}. */
+  type Options = Pick<claim.Options, 'now' | 'staleBefore'>
+
+  /** Pending-work snapshot. */
+  type Result = {
+    /** Pending obligations, including in-flight claims and scheduled retries. */
+    count: number
+    /** Obligations due now with no live claim; the sweeper's backlog. */
+    dueCount: number
+    /** Oldest pending staging timestamp, or null when none. */
+    oldestAt: string | null
+    /** Oldest due-and-unclaimed retry time, or null when none; drives the stall alert. */
+    oldestDueAt: string | null
+  }
+}
+
+/**
+ * Deletes terminal dedupe rows past their retention deadline. Pending rows
+ * never expire: owed work drains only through terminal transitions
+ * (delivery, permanent failure, or the skip after auto-disable).
+ */
+export async function pruneExpired(db: Db.Db, now: string): Promise<number> {
+  const result = await db.kysely
+    .deleteFrom('webhook_queue_events')
+    .where('expiresAt', '<=', now)
+    .where('status', '!=', 'pending')
+    .executeTakeFirst()
+  return Number(result.numDeletedRows)
+}

package/src/db/tables/webhookSubscriptions.ts

@@ -13,6 +13,21 @@ export type Table = db_Schema.WebhookSubscription
 /** A stored webhook-subscription row. */
 export type Record = Selectable<Table>
 
+/** Reads one live subscription by id for an internal Queue delivery. */
+export async function getById(
+  db: Db.Db,
+  id: string,
+  now: string,
+): Promise<Webhooks.Subscription | null> {
+  const row = await db.kysely
+    .selectFrom('webhook_subscriptions')
+    .selectAll()
+    .where('id', '=', id)
+    .where(live(now))
+    .executeTakeFirst()
+  return row ? toSubscription(row) : null
+}
+
 /** Maps an owner union to its stored `ownerId` column value. */
 export function ownerId(owner: Webhooks.Owner): string {
   return owner.type === 'api_key' ? owner.orgId : owner.payer
@@ -273,20 +288,21 @@ export async function remove(
 
 /**
  * Lists live `active` subscriptions for a `(chainId, eventType)` stream — the
- * poller's per-stream lookup, served by the partial status index.
+ * poller's per-stream lookup, served by the partial status index. Each row
+ * carries its poller cursor so the poller avoids a per-subscription read.
  *
  * @param db - The database.
  * @param chainId - Chain the stream belongs to.
  * @param eventType - Event type of the stream.
  * @param now - ISO timestamp expiry is evaluated against.
- * @returns The subscriptions, in id (chronological) order.
+ * @returns Cursor-and-subscription pairs, in id (chronological) order.
  */
 export async function listActive(
   db: Db.Db,
   chainId: number,
   eventType: Webhooks.EventType,
   now: string,
-): Promise<Webhooks.Subscription[]> {
+): Promise<{ cursor: string | null; subscription: Webhooks.Subscription }[]> {
   const rows = await db.kysely
     .selectFrom('webhook_subscriptions')
     .selectAll()
@@ -296,7 +312,69 @@ export async function listActive(
     .where(live(now))
     .orderBy('id')
     .execute()
-  return rows.map(toSubscription)
+  return rows.map((row) => ({ cursor: row.pollerCursor, subscription: toSubscription(row) }))
+}
+
+/**
+ * Lists every live `active` subscription on a chain in one read. The poller
+ * groups the rows by event type so empty streams cost no extra database reads.
+ *
+ * @param db - The database.
+ * @param options - Chain, configured event types, and expiry time.
+ * @returns Cursor-and-subscription pairs ordered by event type, then id.
+ */
+export async function listActiveForChain(
+  db: Db.Db,
+  options: listActiveForChain.Options,
+): Promise<{ cursor: string | null; subscription: Webhooks.Subscription }[]> {
+  if (options.eventTypes.length === 0) return []
+  const rows = await db.kysely
+    .selectFrom('webhook_subscriptions')
+    .selectAll()
+    .where('chainId', '=', options.chainId)
+    .where('eventType', 'in', options.eventTypes)
+    .where('status', '=', 'active')
+    .where(live(options.now))
+    .orderBy('eventType')
+    .orderBy('id')
+    .execute()
+  return rows.map((row) => ({ cursor: row.pollerCursor, subscription: toSubscription(row) }))
+}
+
+export declare namespace listActiveForChain {
+  /** Filters for the poller's chain-level subscription read. */
+  type Options = {
+    /** Chain whose subscriptions to read. */
+    chainId: number
+    /** Event types configured for this poller tick. */
+    eventTypes: readonly Webhooks.EventType[]
+    /** ISO timestamp expiry is evaluated against. */
+    now: string
+  }
+}
+
+/**
+ * Reads the poller cursors of every live `active` subscription on a chain,
+ * across event types. Feeds the cron's cursor-freshness fallback gate.
+ *
+ * @param db - The database.
+ * @param chainId - Chain whose cursors to read.
+ * @param now - ISO timestamp expiry is evaluated against.
+ * @returns The non-null cursors, unordered.
+ */
+export async function listActiveCursors(
+  db: Db.Db,
+  chainId: number,
+  now: string,
+): Promise<string[]> {
+  const rows = await db.kysely
+    .selectFrom('webhook_subscriptions')
+    .select('pollerCursor')
+    .where('chainId', '=', chainId)
+    .where('status', '=', 'active')
+    .where(live(now))
+    .execute()
+  return rows.flatMap((row) => (row.pollerCursor === null ? [] : [row.pollerCursor]))
 }
 
 /**
@@ -367,6 +445,55 @@ export declare namespace compareAndSetCursor {
   }
 }
 
+/**
+ * Writes multiple poller cursors when each stored value still matches the
+ * caller's observed value.
+ *
+ * @param db - The database.
+ * @param options - Candidate cursor writes.
+ * @returns Subscription ids whose cursors were updated.
+ */
+export async function compareAndSetCursors(
+  db: Db.Db,
+  options: compareAndSetCursors.Options,
+): Promise<Set<string>> {
+  if (options.cursors.length === 0) return new Set()
+  const values = JSON.stringify(options.cursors)
+  type Candidate = {
+    cursor: string
+    expectedCursor: string | null
+    subscriptionId: string
+  }
+  const result = await db.kysely
+    .with(
+      'candidates',
+      () => sql<Candidate>`(
+      SELECT
+        candidate."subscriptionId" AS subscription_id,
+        candidate."expectedCursor" AS expected_cursor,
+        candidate.cursor
+      FROM jsonb_to_recordset(${values}::jsonb)
+        AS candidate("subscriptionId" text, "expectedCursor" text, cursor text)
+    )`,
+    )
+    .updateTable('webhook_subscriptions as subscription')
+    .from('candidates as candidate')
+    .set((eb) => ({ pollerCursor: eb.ref('candidate.cursor') }))
+    .whereRef('subscription.id', '=', 'candidate.subscriptionId')
+    .whereRef('subscription.pollerCursor', 'is not distinct from', 'candidate.expectedCursor')
+    .returning('subscription.id as subscriptionId')
+    .execute()
+  return new Set(result.map((row) => row.subscriptionId))
+}
+
+export declare namespace compareAndSetCursors {
+  /** Options for {@link compareAndSetCursors}. */
+  type Options = {
+    /** Candidate cursor writes. */
+    cursors: readonly compareAndSetCursor.Options[]
+  }
+}
+
 /**
  * Records a successful delivery: resets the consecutive-failure counter and
  * stamps `lastDeliveryAt`.