spine-framework 0.4.3 → 0.5.0-rc.4

package/.framework/tests/unit/pipeline-runner.test.ts

@@ -30,7 +30,7 @@ vi.mock('../../functions/ai-interactions.ts', () => ({
 
 import { adminDb } from '../../functions/_shared/db.ts'
 import { emitAudit } from '../../functions/_shared/audit.ts'
-import { runPipeline } from '../../functions/_shared/pipeline-runner.ts'
+import { runPipeline, sweepStaleExecutions } from '../../functions/_shared/pipeline-runner.ts'
 // CIL[DOC-VECTOR type="context" content="purpose='Test setup and imports for pipeline runner tests'; No verified external callers; imports: .framework/functions/_shared/middleware.ts,.framework/functions/_shared/db.ts,.framework/functions/_shared/audit.ts"]
 // CIL[DOC-VECTOR type="blast_radius" content="No direct caller data from CIL graph; local impact only"]
 // CIL[DOC-VECTOR type="configuration" content="No environment or build configuration dependencies identified"]
@@ -276,6 +276,105 @@ describe('runPipeline — lifecycle', () => {
       expect.objectContaining({ pipeline_id: 'pipe-1' })
     )
   })
+
+  it('claims a pre-created queued execution row when opts.executionId is supplied', async () => {
+    mockPipeline([])
+    const PRE_CREATED_ID = 'exec-pre-created-1'
+    const updateSpy = vi.fn().mockReturnThis()
+    const eqSpy = vi.fn().mockReturnThis()
+    const claimSelectSpy = vi.fn().mockResolvedValue({ data: [{ id: PRE_CREATED_ID }], error: null })
+    const claimChain = { update: updateSpy, eq: eqSpy, select: claimSelectSpy, insert: vi.fn().mockReturnThis() }
+    vi.mocked(adminDb.from).mockImplementation((table: string) => {
+      if (table === 'pipelines') {
+        return {
+          select: vi.fn().mockReturnThis(),
+          eq: vi.fn().mockReturnThis(),
+          single: vi.fn().mockResolvedValue({ data: { id: 'pipe-1', stages: [], is_active: true }, error: null })
+        } as any
+      }
+      if (table === 'pipeline_executions') return claimChain as any
+      return {
+        select: vi.fn().mockReturnThis(),
+        insert: vi.fn().mockReturnThis(),
+        update: vi.fn().mockReturnThis(),
+        eq: vi.fn().mockReturnThis(),
+        single: vi.fn().mockResolvedValue({ data: null, error: null })
+      } as any
+    })
+
+    const result = await runPipeline('pipe-1', { x: 1 }, makeCtx(), { executionId: PRE_CREATED_ID })
+    expect(result.executionId).toBe(PRE_CREATED_ID)
+    expect(result.status).toBe('completed')
+    // The claim path must call update, not insert.
+    expect(updateSpy).toHaveBeenCalledWith(
+      expect.objectContaining({ status: 'running' })
+    )
+    expect(eqSpy).toHaveBeenCalledWith('id', PRE_CREATED_ID)
+    expect(eqSpy).toHaveBeenCalledWith('status', 'queued')
+    // and it must NOT have inserted a fresh row
+    expect(claimChain.insert).not.toHaveBeenCalled()
+  })
+
+  it('inserts with the same id when the queued row is absent (fallback path)', async () => {
+    const PRE_CREATED_ID = 'exec-missing-1'
+    const insertSpy = vi.fn().mockReturnThis()
+    const singleSpy = vi.fn().mockResolvedValue({ data: { id: PRE_CREATED_ID }, error: null })
+    vi.mocked(adminDb.from).mockImplementation((table: string) => {
+      if (table === 'pipelines') {
+        return {
+          select: vi.fn().mockReturnThis(),
+          eq: vi.fn().mockReturnThis(),
+          single: vi.fn().mockResolvedValue({ data: { id: 'pipe-1', stages: [], is_active: true }, error: null })
+        } as any
+      }
+      if (table === 'pipeline_executions') {
+        const chain: any = {
+          update: vi.fn().mockReturnThis(),
+          eq: vi.fn().mockReturnThis(),
+          select: vi.fn().mockResolvedValue({ data: [], error: null }),   // claim finds no rows
+          insert: insertSpy,
+          single: singleSpy
+        }
+        // insert().select().single() — select must chain to an object with single
+        insertSpy.mockImplementation(() => ({
+          select: vi.fn().mockImplementation(() => ({ single: singleSpy }))
+        }))
+        return chain
+      }
+      return {
+        select: vi.fn().mockReturnThis(),
+        update: vi.fn().mockReturnThis(),
+        eq: vi.fn().mockReturnThis(),
+        single: vi.fn().mockResolvedValue({ data: null, error: null })
+      } as any
+    })
+
+    const result = await runPipeline('pipe-1', {}, makeCtx(), { executionId: PRE_CREATED_ID })
+    expect(result.executionId).toBe(PRE_CREATED_ID)
+    expect(insertSpy).toHaveBeenCalledWith(
+      expect.objectContaining({ id: PRE_CREATED_ID, status: 'running' })
+    )
+  })
+
+  it('throws when claiming a queued execution row fails', async () => {
+    vi.mocked(adminDb.from).mockImplementation((table: string) => {
+      if (table === 'pipelines') {
+        return {
+          select: vi.fn().mockReturnThis(),
+          eq: vi.fn().mockReturnThis(),
+          single: vi.fn().mockResolvedValue({ data: { id: 'pipe-1', stages: [], is_active: true }, error: null })
+        } as any
+      }
+      return {
+        update: vi.fn().mockReturnThis(),
+        eq: vi.fn().mockReturnThis(),
+        select: vi.fn().mockResolvedValue({ data: null, error: { message: 'row not found' } })
+      } as any
+    })
+    await expect(
+      runPipeline('pipe-1', {}, makeCtx(), { executionId: 'missing-exec' })
+    ).rejects.toThrow('Failed to claim queued execution missing-exec')
+  })
 })
 // CIL[DOC-VECTOR type="context" content="purpose='Test runPipeline lifecycle and error handling'; No verified external callers; imports: .framework/functions/_shared/middleware.ts,.framework/functions/_shared/db.ts,.framework/functions/_shared/audit.ts"]
 // CIL[DOC-VECTOR type="blast_radius" content="No direct caller data from CIL graph; local impact only"]
@@ -920,3 +1019,56 @@ describe('send_email — recipient resolution + outbound send', () => {
     expect(result.stages[0].output.skipped).toMatch(/not configured/)
   })
 })
+
+// CIL[BEGIN stable_id="~/.framework/tests/unit/pipeline-runner.test.ts:describe_sweepStaleExecutions:block" type="block" identity="describe_sweepStaleExecutions" parent="none"]
+// CIL[DOC-MACRO purpose="Test stale execution sweeper"]
+// CIL[DOC-RESPONSIBILITY responsibility="Verify sweepStaleExecutions marks stale rows failed and leaves fresh rows alone"]
+// CIL[DOC-CONTEXT module="tests/unit/pipeline-runner" layer="test-unit" related_files=".framework/functions/_shared/db.ts,.framework/functions/_shared/pipeline-runner.ts"]
+// CIL[DOC-CONTRACT inputs="none" outputs="test assertions" errors="none" side_effects="Tests sweepStaleExecutions"]
+// CIL[DOC-MICRO logic_summary="Mocks adminDb.from('pipeline_executions').update chain and verifies the correct statuses and cutoff are targeted" notable_decisions="[]"]
+// CIL[DOC-CONFIDENCE pass="first" score="0.9" flags="none"]
+// CIL[DOC-STABILITY stability="stable"]
+describe('sweepStaleExecutions', () => {
+  beforeEach(() => vi.clearAllMocks())
+
+  it('marks running/pending/queued rows older than the stale window as failed', async () => {
+    const updateSpy = vi.fn().mockReturnThis()
+    const inSpy = vi.fn().mockReturnThis()
+    const ltSpy = vi.fn().mockResolvedValue({ error: null })
+    vi.mocked(adminDb.from).mockReturnValue({
+      update: updateSpy,
+      in: inSpy,
+      lt: ltSpy
+    } as any)
+
+    await sweepStaleExecutions()
+
+    expect(adminDb.from).toHaveBeenCalledWith('pipeline_executions')
+    expect(updateSpy).toHaveBeenCalledWith(
+      expect.objectContaining({ status: 'failed' })
+    )
+    expect(inSpy).toHaveBeenCalledWith('status', ['running', 'pending', 'queued'])
+    // lt cutoff must be ~10 minutes ago
+    const cutoffArg = ltSpy.mock.calls[0][1] as string
+    const cutoffMs = new Date(cutoffArg).getTime()
+    const expectedMs = Date.now() - 10 * 60 * 1000
+    expect(Math.abs(cutoffMs - expectedMs)).toBeLessThan(5000)
+  })
+
+  it('logs a warning but does not throw when the sweep update fails', async () => {
+    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+    vi.mocked(adminDb.from).mockReturnValue({
+      update: vi.fn().mockReturnThis(),
+      in: vi.fn().mockReturnThis(),
+      lt: vi.fn().mockResolvedValue({ error: { message: 'DB gone' } })
+    } as any)
+
+    await expect(sweepStaleExecutions()).resolves.toBeUndefined()
+    expect(warnSpy).toHaveBeenCalledWith(
+      expect.stringContaining('pipeline_executions stale sweep failed'),
+      'DB gone'
+    )
+    warnSpy.mockRestore()
+  })
+})
+// CIL[END stable_id="~/.framework/tests/unit/pipeline-runner.test.ts:describe_sweepStaleExecutions:block"]

package/.framework/tests/unit/principal-cache.test.ts

@@ -135,4 +135,19 @@ describe('resolvePrincipal human cache — mid-registration guard', () => {
     expect(mockPeopleSingle).toHaveBeenCalledTimes(1)
   })
 })
+
+describe('internal invoke secret', () => {
+  it('resolves a bare x-internal-secret to the system principal', async () => {
+    vi.stubEnv('INTERNAL_INVOKE_SECRET', 'test-secret')
+    const p = await resolvePrincipal({ headers: { 'x-internal-secret': 'test-secret' } })
+    expect(p.id).toBe('system')
+    expect(p.machineType).toBe('internal')
+  })
+
+  it('does not trust a wrong secret', async () => {
+    vi.stubEnv('INTERNAL_INVOKE_SECRET', 'test-secret')
+    const p = await resolvePrincipal({ headers: { 'x-internal-secret': 'wrong' } })
+    expect(p.id).toBe('anonymous')
+  })
+})
 // CIL[END stable_id="~/.framework/tests/unit/principal-cache.test.ts:module_header:module_header"]

package/.framework/tests/unit/system-context-denial.test.ts

@@ -0,0 +1,145 @@
+import { describe, it, expect, vi } from 'vitest'
+
+// F-051: requireSystemContextWithAudit's ctx overload used to return a
+// Netlify response object on denial; handlers returned it as `result`, so
+// createHandler wrapped it as {data:{statusCode:403}} under HTTP 200. It now
+// throws a statusCode-carrying error which the catch path maps to a real
+// HTTP status. These tests exercise the real createHandler end-to-end for
+// security.ts and tests.ts.
+
+const principalRef = vi.hoisted(() => ({ principal: null as any }))
+
+vi.mock('../../functions/_shared/principal.ts', () => ({
+  resolvePrincipal: vi.fn(async () => principalRef.principal),
+  isSystemAdmin: (p: any) =>
+    !!p?.roles?.some((r: any) => r === 'system_admin' || r?.slug === 'system_admin'),
+  getPrincipalDb: () => ({}),
+}))
+
+vi.mock('../../functions/_shared/db.ts', () => ({
+  adminDb: {
+    from: () => {
+      const builder: any = {}
+      for (const m of [
+        'select', 'eq', 'in', 'is', 'ilike', 'filter', 'not', 'order', 'limit',
+        'update', 'insert', 'upsert', 'delete', 'neq', 'or', 'contains', 'gte',
+        'lte', 'lt', 'range', 'textSearch',
+      ]) {
+        builder[m] = () => builder
+      }
+      builder.single = () => Promise.resolve({ data: null, error: null })
+      builder.maybeSingle = () => Promise.resolve({ data: null, error: null })
+      builder.then = (res: any, rej: any) =>
+        Promise.resolve({ data: [], error: null }).then(res, rej)
+      return builder
+    },
+    rpc: vi.fn(async () => ({ data: null, error: null })),
+    auth: { admin: {} },
+  },
+  getUserDb: vi.fn(() => ({})),
+}))
+
+vi.mock('../../functions/_shared/audit.ts', () => ({
+  emitAudit: vi.fn(async () => {}),
+  emitSpan: vi.fn(async () => {}),
+  emitLog: vi.fn(async () => {}),
+  emitActivity: vi.fn(async () => {}),
+}))
+
+vi.mock('../../functions/_shared/security.ts', () => ({
+  isBlockedRequest: vi.fn(async () => null),
+  isBlockedIp: vi.fn(async () => false),
+  touchPersonGeo: vi.fn(async () => {}),
+}))
+
+vi.mock('../../functions/_shared/geo.ts', () => ({
+  getClientGeo: vi.fn(() => ({ ip: '127.0.0.1' })),
+}))
+
+import { requireSystemContextWithAudit } from '../../functions/_shared/middleware'
+import { handler as securityHandler } from '../../functions/security'
+import { handler as testsHandler } from '../../functions/tests'
+
+const memberPrincipal = {
+  id: 'p-member',
+  type: 'human',
+  accountId: 'acct-1',
+  roles: ['member'],
+  provenance: { sourceType: 'jwt', createdBy: null, invokedAt: new Date().toISOString() },
+}
+
+const adminPrincipal = {
+  ...memberPrincipal,
+  id: 'p-admin',
+  roles: ['system_admin'],
+}
+
+const event = (queryStringParameters: Record<string, string> = {}) => ({
+  headers: {},
+  httpMethod: 'GET',
+  path: '/api/x',
+  queryStringParameters,
+  body: null,
+})
+
+describe('requireSystemContextWithAudit ctx overload (F-051)', () => {
+  it('throws a statusCode 403 error for a non-admin principal', () => {
+    const ctx = { principal: memberPrincipal, accountId: 'acct-1' }
+    try {
+      requireSystemContextWithAudit(ctx as any)
+      expect.unreachable('should have thrown')
+    } catch (err: any) {
+      expect(err).toBeInstanceOf(Error)
+      expect(err.statusCode).toBe(403)
+      expect(err.message).toBe('System context required')
+    }
+  })
+
+  it('throws a statusCode 401 error for an anonymous principal', () => {
+    const ctx = { principal: { id: 'anonymous', type: 'machine' }, accountId: null }
+    try {
+      requireSystemContextWithAudit(ctx as any)
+      expect.unreachable('should have thrown')
+    } catch (err: any) {
+      expect(err.statusCode).toBe(401)
+      expect(err.message).toBe('Authentication required')
+    }
+  })
+
+  it('returns null and stamps triggeredBy for a system_admin principal', () => {
+    const ctx: any = { principal: { ...adminPrincipal }, accountId: 'acct-1' }
+    expect(requireSystemContextWithAudit(ctx, 'suite')).toBeNull()
+    expect(ctx.triggeredBy).toBe('suite')
+  })
+})
+
+describe('denied responses are real HTTP errors (F-051)', () => {
+  it('tests.ts returns HTTP 403 — not a 200-wrapped response object', async () => {
+    principalRef.principal = memberPrincipal
+    const res = await testsHandler(event({ action: 'list' }), {})
+    expect(res.statusCode).toBe(403)
+    const body = JSON.parse(res.body)
+    expect(body.error).toBe('System context required')
+    expect(body.data).toBeNull()
+    // Regression shape: previously {statusCode:200, body:{data:{statusCode:403}}}
+    expect(body.data?.statusCode).toBeUndefined()
+  })
+
+  it('security.ts returns HTTP 403 — not a 200-wrapped response object', async () => {
+    principalRef.principal = memberPrincipal
+    const res = await securityHandler(event({ action: 'blocklist' }), {})
+    expect(res.statusCode).toBe(403)
+    const body = JSON.parse(res.body)
+    expect(body.error).toBe('System context required')
+    expect(body.data).toBeNull()
+  })
+
+  it('tests.ts still serves a system_admin principal with HTTP 200', async () => {
+    principalRef.principal = adminPrincipal
+    const res = await testsHandler(event({ action: 'list' }), {})
+    expect(res.statusCode).toBe(200)
+    const body = JSON.parse(res.body)
+    expect(body.error).toBeNull()
+    expect(body.data.data).toEqual([])
+  })
+})

package/.framework/tests/unit/system-cron.test.ts

@@ -0,0 +1,240 @@
+// Tests for system-cron's per-tick maintenance blocks (F-052).
+// Regression: when get_due_schedules returns zero rows the handler used to
+// return early, skipping due timers, threshold alerts, delivery retries,
+// and log cleanup. These tests exercise the handler with all shared deps
+// mocked; createHandler is stubbed to identity so a fabricated
+// RequestContext flows straight into the handler body.
+import { describe, it, expect, vi, beforeEach } from 'vitest'
+
+const {
+  rpcMock,
+  fromMock,
+  emitAuditMock,
+  runPipelineMock,
+  sweepMock,
+  retryMock,
+} = vi.hoisted(() => ({
+  rpcMock: vi.fn(),
+  fromMock: vi.fn(),
+  emitAuditMock: vi.fn(),
+  runPipelineMock: vi.fn(),
+  sweepMock: vi.fn(),
+  retryMock: vi.fn(),
+}))
+
+vi.mock('../../functions/_shared/middleware.ts', () => ({
+  createHandler: (fn: any) => fn,
+}))
+
+vi.mock('../../functions/_shared/db.ts', () => ({
+  adminDb: { rpc: rpcMock, from: fromMock },
+}))
+
+vi.mock('../../functions/_shared/audit.ts', () => ({
+  emitAudit: emitAuditMock,
+}))
+
+vi.mock('../../functions/_shared/pipeline-runner.ts', () => ({
+  runPipeline: runPipelineMock,
+  sweepStaleExecutions: sweepMock,
+}))
+
+vi.mock('../../functions/_shared/retry-deliveries.ts', () => ({
+  retryPendingDeliveries: retryMock,
+}))
+
+import { handler } from '../../functions/system-cron.ts'
+
+function makeCtx() {
+  return {
+    requestId: 'test-req',
+    principal: {
+      id: 'system',
+      type: 'machine',
+      machineType: 'internal',
+      accountId: null,
+      scopes: ['*:*'],
+      isInternal: true,
+    },
+    db: {},
+    accountId: null,
+    appId: null,
+    query: {},
+    requestPath: '/.netlify/functions/system-cron',
+    headers: {},
+  }
+}
+
+// Per-RPC-name responses; anything not listed resolves to an empty list.
+function rpcImpl(overrides: Record<string, any> = {}) {
+  return (name: string) => {
+    if (name in overrides) {
+      const v = overrides[name]
+      return v instanceof Error ? Promise.reject(v) : Promise.resolve(v)
+    }
+    return Promise.resolve({ data: [], error: null })
+  }
+}
+
+// Chainable thenable query builder; terminal result is keyed by table name.
+function fromImpl(tableData: Record<string, any> = {}) {
+  return (table: string) => {
+    const result = table in tableData ? tableData[table] : { data: [], error: null }
+    const chain: any = {}
+    for (const m of ['select', 'eq', 'neq', 'lt', 'gt', 'or', 'in', 'order', 'limit', 'insert', 'update', 'single', 'maybeSingle']) {
+      chain[m] = vi.fn(() => chain)
+    }
+    chain.then = (onF: any, onR: any) => Promise.resolve(result).then(onF, onR)
+    return chain
+  }
+}
+
+const RETRY_SUMMARY = {
+  outbound_retried: 0,
+  outbound_succeeded: 0,
+  outbound_failed: 0,
+  inbound_retried: 0,
+  inbound_succeeded: 0,
+  inbound_failed: 0,
+  errors: [],
+}
+
+describe('system-cron', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    sweepMock.mockResolvedValue(undefined)
+    retryMock.mockResolvedValue(RETRY_SUMMARY)
+  })
+
+  it('runs per-tick maintenance when no schedules are due', async () => {
+    rpcMock.mockImplementation(rpcImpl({
+      get_due_schedules: { data: [], error: null },
+      get_due_timers: { data: [], error: null },
+    }))
+    fromMock.mockImplementation(fromImpl({
+      threshold_alerts: { data: [], error: null },
+    }))
+
+    const res = await handler(makeCtx())
+    expect(res.statusCode).toBe(200)
+
+    // Every per-tick block still ran despite the empty schedule list.
+    expect(sweepMock).toHaveBeenCalledOnce()
+    expect(rpcMock).toHaveBeenCalledWith('get_due_timers', expect.anything())
+    expect(fromMock).toHaveBeenCalledWith('threshold_alerts')
+    expect(retryMock).toHaveBeenCalledOnce()
+
+    const body = JSON.parse(res.body)
+    expect(body.schedules_due).toBe(0)
+    expect(body.executed).toBe(0)
+    expect(body.results).toEqual([])
+    expect(body.thresholds_evaluated).toBe(0)
+    expect(body.deliveries_retried).toEqual(RETRY_SUMMARY)
+  })
+
+  it('executes due timers on a tick with no due schedules', async () => {
+    const timer = {
+      timer_id: 't1',
+      pipeline_id: 'p1',
+      account_id: 'a1',
+      name: 'nightly',
+      execution_count: 3,
+      created_by: 'u1',
+      app_id: null,
+    }
+    rpcMock.mockImplementation(rpcImpl({
+      get_due_schedules: { data: [], error: null },
+      get_due_timers: { data: [timer], error: null },
+      update_timer_after_run: { data: null, error: null },
+    }))
+    fromMock.mockImplementation(fromImpl({
+      threshold_alerts: { data: [], error: null },
+    }))
+    runPipelineMock.mockResolvedValue({ status: 'completed' })
+
+    const res = await handler(makeCtx())
+    expect(res.statusCode).toBe(200)
+
+    expect(runPipelineMock).toHaveBeenCalledWith(
+      'p1',
+      expect.objectContaining({ timer_id: 't1' }),
+      expect.objectContaining({ accountId: 'a1' }),
+    )
+    expect(rpcMock).toHaveBeenCalledWith('update_timer_after_run',
+      expect.objectContaining({ p_timer_id: 't1', p_success: true }))
+
+    const body = JSON.parse(res.body)
+    expect(body.schedules_due).toBe(0)
+    expect(body.executed).toBe(1)
+    expect(body.success).toBe(1)
+    expect(body.results[0].scheduleId).toBe('t1')
+    expect(retryMock).toHaveBeenCalledOnce()
+  })
+
+  it('still reaches delivery retries when the timer block throws', async () => {
+    rpcMock.mockImplementation(rpcImpl({
+      get_due_schedules: { data: [], error: null },
+      get_due_timers: new Error('rpc down'),
+    }))
+    fromMock.mockImplementation(fromImpl({
+      threshold_alerts: { data: [], error: null },
+    }))
+
+    const res = await handler(makeCtx())
+    expect(res.statusCode).toBe(200)
+    expect(retryMock).toHaveBeenCalledOnce()
+  })
+
+  it('still processes due schedules and then runs maintenance', async () => {
+    const schedule = {
+      schedule_id: 's1',
+      action_id: 'act1',
+      account_id: 'a1',
+      machine_principal_id: 'm1',
+      delegated_scopes: ['*:*'],
+      config: {},
+    }
+    const action = {
+      id: 'act1',
+      handler: 'send_email',
+      handler_module: 'functions',
+      config: {},
+      required_scopes: ['*:*'],
+    }
+    const machine = {
+      id: 'm1',
+      account_id: 'a1',
+      scopes: ['*:*'],
+      machine_type: 'internal',
+      is_internal: true,
+      created_by: 'u1',
+    }
+
+    rpcMock.mockImplementation(rpcImpl({
+      get_due_schedules: { data: [schedule], error: null },
+      get_due_timers: { data: [], error: null },
+      validate_schedule_creator: { data: { is_valid: true }, error: null },
+      update_schedule_after_run: { data: null, error: null },
+    }))
+    fromMock.mockImplementation(fromImpl({
+      actions: { data: action, error: null },
+      api_keys: { data: machine, error: null },
+      threshold_alerts: { data: [], error: null },
+      schedule_executions: { data: null, error: null },
+    }))
+
+    const res = await handler(makeCtx())
+    expect(res.statusCode).toBe(200)
+
+    const body = JSON.parse(res.body)
+    expect(body.schedules_due).toBe(1)
+    expect(body.executed).toBe(1)
+    expect(body.success).toBe(1)
+    expect(body.results[0].scheduleId).toBe('s1')
+
+    // Maintenance blocks still ran after schedule processing.
+    expect(rpcMock).toHaveBeenCalledWith('get_due_timers', expect.anything())
+    expect(fromMock).toHaveBeenCalledWith('threshold_alerts')
+    expect(retryMock).toHaveBeenCalledOnce()
+  })
+})