tapimo 0.15.0 → 0.18.0

package/src/ApiKeys.test.ts

@@ -1,9 +1,12 @@
+import { sql } from 'kysely'
+
 import * as ApiKeys from './ApiKeys.js'
 import * as ApiKeyAdmissions from './db/tables/apiKeyAdmissions.js'
 import * as ApiKeyOwnerTombstones from './db/tables/apiKeyOwnerTombstones.js'
 import * as ApiKeyRevocations from './db/tables/apiKeyRevocations.js'
 import * as Organizations from './db/tables/organizations.js'
 import * as Projects from './db/tables/projects.js'
+import * as RoutesPolicies from './db/tables/routesPolicies.js'
 import * as Store from './internal/Store.js'
 import * as TestApp from '../test/App.js'
 
@@ -214,6 +217,234 @@ describe('markOwnerDeleting', () => {
 })
 
 describe('mintBounded', () => {
+  test.each(['routes:write', '*'])(
+    'rejects delegation after the locked source loses %s',
+    async (scope) => {
+      const db = TestApp.database()
+      const state = Store.memory()
+      const org = await Organizations.create(db, { name: 'Delegation race' })
+      const project = await Projects.create(db, { name: 'Delegation', orgId: org.id })
+      const { record: source } = await ApiKeys.mintBounded(db, state, {
+        orgId: org.id,
+        projectId: project.id,
+        scopes: [scope],
+      })
+      // The handler authorized this snapshot before a restrictive edit committed.
+      await ApiKeys.updateBounded(db, state, source.id, { scopes: ['routes:read'] })
+      await expect(
+        ApiKeys.mintBounded(
+          db,
+          state,
+          {
+            environment: source.environment,
+            orgId: source.orgId,
+            projectId: source.projectId,
+            scopes: ['routes:write'],
+          },
+          {
+            policySourceApiKeyId: source.id,
+            sourceScopes: ['routes:write'],
+          },
+        ),
+      ).rejects.toBeInstanceOf(RoutesPolicies.InvalidScopeError)
+      expect((await ApiKeys.listByOrg(state, org.id)).map(({ id }) => id)).toEqual([source.id])
+    },
+  )
+
+  test('copies current locked source restrictions instead of the earlier handler snapshot', async () => {
+    const db = TestApp.database()
+    const state = Store.memory()
+    const org = await Organizations.create(db, { name: 'Restriction race' })
+    const project = await Projects.create(db, { name: 'Delegation', orgId: org.id })
+    const { record: source } = await ApiKeys.mintBounded(db, state, {
+      allowedIps: [],
+      expiresAt: new Date(Date.now() + 120_000).toISOString(),
+      orgId: org.id,
+      projectId: project.id,
+      rateLimits: { '*': { limit: 10_000, period: 'minute' } },
+      scopes: ['routes:write'],
+    })
+    const updated = await ApiKeys.updateBounded(db, state, source.id, {
+      allowedIps: ['203.0.113.7'],
+      rateLimits: { '*': { limit: 10, period: 'minute' } },
+    })
+    const { record } = await ApiKeys.mintBounded(
+      db,
+      state,
+      {
+        allowedIps: source.allowedIps,
+        environment: source.environment,
+        expiresAt: source.expiresAt,
+        orgId: source.orgId,
+        projectId: source.projectId,
+        rateLimits: source.rateLimits,
+        scopes: ['routes:write'],
+      },
+      {
+        policySourceApiKeyId: source.id,
+        sourceScopes: ['routes:write'],
+      },
+    )
+    expect(record.allowedIps).toEqual(updated!.allowedIps)
+    expect(record.expiresAt).toBe(updated!.expiresAt)
+    expect(record.rateLimits).toEqual(updated!.rateLimits)
+  })
+
+  test('releases an unpublished key admission when its KV index disappears before publication', async () => {
+    const database = TestApp.databaseFactory()
+    const db = database()
+    const state = Store.memory()
+    const organization = await Organizations.create(db, { name: 'Publication failure' })
+    await sql
+      .raw(`
+      CREATE FUNCTION pause_admission_commit() RETURNS trigger AS $$
+      BEGIN
+        PERFORM pg_advisory_xact_lock(12813337);
+        RETURN NEW;
+      END;
+      $$ LANGUAGE plpgsql;
+      CREATE CONSTRAINT TRIGGER pause_admission_commit
+        AFTER INSERT ON api_key_admissions DEFERRABLE INITIALLY DEFERRED
+        FOR EACH ROW EXECUTE FUNCTION pause_admission_commit();
+    `)
+      .execute(db.kysely)
+    const locked = Promise.withResolvers<void>()
+    const release = Promise.withResolvers<void>()
+    const blocker = database().transaction(async (tx) => {
+      await sql`SELECT pg_advisory_xact_lock(12813337)`.execute(tx.kysely)
+      locked.resolve()
+      await release.promise
+    })
+    await locked.promise
+    const minted = ApiKeys.mintBounded(database(), state, {
+      orgId: organization.id,
+      scopes: ['routes:write'],
+    })
+    const outcome = Promise.allSettled([minted])
+    try {
+      await expect
+        .poll(async () => {
+          const { rows } = await sql<{ waiting: boolean }>`select exists (
+          select 1 from pg_locks where locktype = 'advisory' and objid = 12813337 and not granted
+        ) as waiting`.execute(db.kysely)
+          return rows[0]!.waiting
+        })
+        .toBe(true)
+      const [record] = await ApiKeys.listByOrg(state, organization.id)
+      expect(record?.scopes).toEqual([])
+      await state.delete(`apikey_id:${record!.id}`)
+      release.resolve()
+      expect(await outcome).toMatchObject([{ status: 'rejected' }])
+      expect(await ApiKeyAdmissions.exists(db, record!.id)).toBe(false)
+      expect(await ApiKeyRevocations.exists(db, record!.id)).toBe(true)
+      expect(await ApiKeyAdmissions.matches(db, record!)).toBe(false)
+    } finally {
+      release.resolve()
+      await blocker
+      await outcome
+    }
+  })
+
+  test.each(['current', 'stale'])(
+    'security: rejects a moved rotation source with %s KV',
+    async (kv) => {
+      const db = TestApp.database()
+      const state = Store.memory()
+      const org = await Organizations.create(db, { name: 'Rotation race' })
+      const first = await Projects.create(db, { name: 'Original', orgId: org.id })
+      const second = await Projects.create(db, { name: 'Reassigned', orgId: org.id })
+      const { record: source } = await ApiKeys.mintBounded(db, state, {
+        orgId: org.id,
+        projectId: first.id,
+        scopes: ['routes:write'],
+      })
+      const scope = {
+        apiKeyId: source.id,
+        environment: source.environment,
+        orgId: source.orgId,
+        projectId: source.projectId,
+      }
+      await RoutesPolicies.save(db, { scope, version: 0, policy: { access: [] } })
+
+      // The HTTP rotation handler already read and authorized `source`. A
+      // reassignment can finish before mintBounded takes its policy-copy lock.
+      await ApiKeys.updateBounded(db, state, source.id, { projectId: second.id })
+      expect(await RoutesPolicies.get(db, scope)).toMatchObject({ policy: {}, version: 2 })
+      expect((await ApiKeys.get(state, source.id))?.projectId).toBe(second.id)
+      // KV can lag SQL. Re-create the stale attribution through the KV write
+      // boundary; the authoritative admission must still reject the rotation.
+      if (kv === 'stale') await ApiKeys.update(state, source.id, { projectId: first.id })
+
+      // Copying the now-empty old scope would mint a usable key in the former
+      // project with unrestricted legacy route access instead of the old denial.
+      await expect(
+        ApiKeys.mintBounded(
+          db,
+          state,
+          {
+            environment: source.environment,
+            orgId: source.orgId,
+            projectId: source.projectId,
+            scopes: source.scopes,
+          },
+          { policySourceApiKeyId: source.id },
+        ).then(({ record }) => ({
+          inOriginalProject: record.projectId === first.id,
+          scopes: record.scopes,
+        })),
+      ).rejects.toBeInstanceOf(RoutesPolicies.InvalidScopeError)
+    },
+  )
+
+  test('leaves a replacement unprivileged when policy copy rolls back at SQL commit', async () => {
+    const db = TestApp.database()
+    const state = Store.memory()
+    const organization = await Organizations.create(db, { name: 'Rotation boundary' })
+    const source = await ApiKeys.mintBounded(db, state, {
+      environment: 'sandbox',
+      orgId: organization.id,
+      scopes: ['data:read'],
+    })
+    const sourceScope = {
+      apiKeyId: source.record.id,
+      environment: 'sandbox',
+      orgId: organization.id,
+    } as const
+    await RoutesPolicies.save(db, {
+      policy: { maxAmount: '17' },
+      scope: sourceScope,
+      version: 0,
+    })
+    await installDeferredAdmissionFailure(db, organization.id)
+
+    await expect(
+      ApiKeys.mintBounded(
+        db,
+        state,
+        {
+          environment: 'sandbox',
+          orgId: organization.id,
+          scopes: ['routes:write'],
+        },
+        { policySourceApiKeyId: source.record.id },
+      ),
+    ).rejects.toThrow('deferred admission failure')
+
+    const records = await ApiKeys.listByOrg(state, organization.id)
+    const replacement = records.find(({ id }) => id !== source.record.id)!
+    expect(records.find(({ id }) => id === source.record.id)?.scopes).toEqual(['data:read'])
+    expect(replacement.scopes).toEqual([])
+    await expect(ApiKeyAdmissions.matches(db, replacement)).resolves.toBe(false)
+    await expect(ApiKeyAdmissions.matches(db, source.record)).resolves.toBe(true)
+    expect(await RoutesPolicies.get(db, sourceScope)).toMatchObject({
+      policy: { maxAmount: '17' },
+      version: 1,
+    })
+    expect(
+      await RoutesPolicies.get(db, { ...sourceScope, apiKeyId: replacement.id }),
+    ).toBeUndefined()
+  })
+
   test('rejects an authoritative owner tombstone absent from KV', async () => {
     const db = TestApp.database()
     const state = Store.memory()
@@ -269,8 +500,8 @@ describe('mintBounded', () => {
       await state.put(recordKey, raw)
       await state.put(`apikey_org:${revoked.record.orgId}:${revoked.record.id}`, recordKey)
 
-      const stale = await ApiKeys.resolve(state, revoked.token)
-      expect(stale).not.toBeNull()
+      const stale = revoked.record
+      expect(await ApiKeys.resolve(state, revoked.token)).toBeNull()
       await expect(ApiKeyAdmissions.matches(db, stale!)).resolves.toBe(false)
       await ApiKeys.mintBounded(db, state, {
         orgId: revoked.record.orgId,
@@ -324,8 +555,8 @@ describe('mintBounded', () => {
     await state.put(recordKey, raw)
     await state.put(`apikey_org:${target.record.orgId}:${target.record.id}`, recordKey)
 
-    const stale = await ApiKeys.resolve(state, target.token)
-    expect(stale).not.toBeNull()
+    const stale = target.record
+    expect(await ApiKeys.resolve(state, target.token)).toBeNull()
     await expect(ApiKeyAdmissions.matches(db, stale!)).resolves.toBe(false)
   })
 
@@ -354,11 +585,12 @@ describe('mintBounded', () => {
       await state.delete(`apikey_id:${target.record.id}`)
 
       await expect(ApiKeys.revokeBounded(db, state, target.record.id)).resolves.toBe(true)
-      const stale = await ApiKeys.resolve(state, target.token)
-      expect(stale).not.toBeNull()
+      const stale = target.record
+      expect(await ApiKeys.resolve(state, target.token)).toBeNull()
       await expect(ApiKeyAdmissions.matches(db, stale!)).resolves.toBe(false)
       vi.advanceTimersByTime(ApiKeyRevocations.retentionMs + 1)
       await expect(ApiKeyRevocations.prune(db)).resolves.toBe(0)
+      expect(await ApiKeys.resolve(state, target.token)).toBeNull()
       await expect(ApiKeyAdmissions.matches(db, stale!)).resolves.toBe(false)
       await expect(ApiKeyAdmissions.exists(db, target.record.id)).resolves.toBe(true)
       await expect(
@@ -725,6 +957,249 @@ describe('mintBounded', () => {
 })
 
 describe('updateBounded', () => {
+  test('keeps post-commit scope publication serialized with another reassignment', async () => {
+    const database = TestApp.databaseFactory()
+    const inner = Store.memory()
+    const publishing = Promise.withResolvers<void>()
+    const release = Promise.withResolvers<void>()
+    const state = Store.from({
+      ...inner,
+      async put(key, value, options = {}) {
+        if (key.startsWith('apikey:')) {
+          const record = JSON.parse(value) as ApiKeys.Record
+          if (record.orgId === 'org_first' && record.scopes.includes('*')) {
+            publishing.resolve()
+            await release.promise
+          }
+        }
+        await inner.put(key, value, options)
+      },
+    })
+    const { record, token } = await ApiKeys.mintBounded(database(), state, {
+      orgId: 'org_source',
+      scopes: ['*'],
+    })
+    const first = ApiKeys.updateBounded(database(), state, record.id, { orgId: 'org_first' })
+    const firstResult = Promise.allSettled([first])
+    await publishing.promise
+    let secondFinished = false
+    const second = ApiKeys.updateBounded(database(), state, record.id, {
+      orgId: 'org_second',
+      scopes: ['routes:read'],
+    })
+    const secondResult = Promise.allSettled([second]).then((result) => {
+      secondFinished = true
+      return result
+    })
+    try {
+      // Either the competing move finishes (the bug), or waits for the publishing key lock.
+      await expect
+        .poll(async () => {
+          const { rows } = await sql<{ waiting: boolean }>`select exists (
+            select 1 from pg_locks where locktype = 'advisory' and not granted
+            and classid::bigint = ((hashtextextended(${`api-key-admission-id:${record.id}`}, 0) >> 32) & 4294967295)
+            and objid::bigint = (hashtextextended(${`api-key-admission-id:${record.id}`}, 0) & 4294967295)
+          ) as waiting`.execute(database().kysely)
+          return secondFinished || rows[0]!.waiting
+        })
+        .toBe(true)
+    } finally {
+      release.resolve()
+      await Promise.all([firstResult, secondResult])
+    }
+    expect(await firstResult).toMatchObject([{ status: 'fulfilled' }])
+    expect(await secondResult).toMatchObject([{ status: 'fulfilled' }])
+    const current = await ApiKeys.resolve(inner, token)
+    expect(current).toMatchObject({ orgId: 'org_second', scopes: ['routes:read'] })
+    expect(await ApiKeyAdmissions.matches(database(), current!)).toBe(true)
+  })
+
+  test.each([
+    { scopes: undefined, expected: ['*'] },
+    { scopes: [], expected: [] },
+    { scopes: ['routes:read'], expected: ['routes:read'] },
+  ] as const)(
+    'publishes the latest committed scope intent after another move: %j',
+    async ({ scopes, expected }) => {
+      const db = TestApp.database()
+      const state = Store.memory()
+      const { record, token } = await ApiKeys.mintBounded(db, state, {
+        orgId: 'org_source',
+        scopes: [],
+      })
+      // Seed the durable boundary after a move commits but before its KV publication.
+      await db.transaction(async (tx) => {
+        await ApiKeyAdmissions.lock(tx, record.id)
+        await ApiKeyAdmissions.setPendingScopes(tx, record.id, ['*'])
+      })
+      if (scopes !== undefined) {
+        await ApiKeys.updateBounded(db, state, record.id, { scopes })
+        expect((await ApiKeyAdmissions.get(db, record.id))?.pendingScopes).toBeNull()
+      }
+      await ApiKeys.updateBounded(db, state, record.id, { orgId: 'org_second' })
+      await ApiKeys.updateBounded(db, state, record.id, { orgId: 'org_source' })
+      expect(await ApiKeys.resolve(state, token)).toMatchObject({
+        orgId: 'org_source',
+        scopes: expected,
+      })
+      expect((await ApiKeyAdmissions.get(db, record.id))?.pendingScopes).toBeNull()
+    },
+  )
+
+  test.each(['reassignment', 'rotation'] as const)(
+    'serializes %s with organization deletion without deadlock',
+    async (operation) => {
+      const database = TestApp.databaseFactory()
+      const db = database()
+      const state = Store.memory()
+      const source = await Organizations.create(db, { name: 'Deleting owner' })
+      const target = await Organizations.create(db, { name: 'New owner' })
+      const { record } = await ApiKeys.mintBounded(db, state, {
+        orgId: source.id,
+        scopes: ['routes:read'],
+      })
+      await RoutesPolicies.save(db, {
+        policy: { maxAmount: '0.07' },
+        scope: { apiKeyId: record.id, environment: record.environment, orgId: source.id },
+        version: 0,
+      })
+      const locked = Promise.withResolvers<void>()
+      const release = Promise.withResolvers<void>()
+      const blocker = database().transaction(async (tx) => {
+        await sql`LOCK TABLE sponsored_transactions IN ACCESS EXCLUSIVE MODE`.execute(tx.kysely)
+        locked.resolve()
+        await release.promise
+      })
+      await locked.promise
+      const deletion = Organizations.deleteOrganization(database(), source.id)
+      const deletionResult = Promise.allSettled([deletion])
+      try {
+        // The deletion holds the subsidy lock while its first liability query waits on the table.
+        await expect
+          .poll(async () => {
+            const { rows } = await sql<{ waiting: boolean }>`select exists (
+          select 1 from pg_locks where relation = 'sponsored_transactions'::regclass and not granted
+        ) as waiting`.execute(db.kysely)
+            return rows[0]!.waiting
+          })
+          .toBe(true)
+        const mutation =
+          operation === 'reassignment'
+            ? ApiKeys.updateBounded(database(), state, record.id, { orgId: target.id })
+            : ApiKeys.mintBounded(
+                database(),
+                state,
+                {
+                  orgId: source.id,
+                  scopes: ['routes:read'],
+                },
+                { policySourceApiKeyId: record.id },
+              )
+        const mutationResult = Promise.allSettled([mutation])
+        try {
+          await expect
+            .poll(async () => {
+              const { rows } = await sql<{ waiting: boolean }>`select exists (
+            select 1 from pg_locks where locktype = 'advisory' and not granted
+            and classid::bigint = ((hashtextextended(${`routes-subsidy:${source.id}`}, 0) >> 32) & 4294967295)
+            and objid::bigint = (hashtextextended(${`routes-subsidy:${source.id}`}, 0) & 4294967295)
+          ) as waiting`.execute(db.kysely)
+              return rows[0]!.waiting
+            })
+            .toBe(true)
+        } finally {
+          release.resolve()
+        }
+        expect(await deletionResult).toMatchObject([{ status: 'fulfilled', value: true }])
+        expect(await mutationResult).toMatchObject([
+          { status: 'rejected', reason: { name: 'ApiKeys.OwnerDeletedError' } },
+        ])
+        expect(await ApiKeys.get(state, record.id)).toMatchObject({
+          orgId: source.id,
+          scopes: ['routes:read'],
+        })
+        expect(await ApiKeyAdmissions.get(db, record.id)).toBeUndefined()
+        expect(await ApiKeyOwnerTombstones.isDeleted(db, { orgId: source.id })).toBe(true)
+      } finally {
+        release.resolve()
+        await blocker
+        await deletionResult
+      }
+    },
+  )
+
+  test('preserves scopes on repeated reassignment after a former project is deleted', async () => {
+    const db = TestApp.database()
+    const state = Store.memory()
+    const org = await Organizations.create(db, { name: 'Reassignment' })
+    const first = await Projects.create(db, { name: 'First', orgId: org.id })
+    const second = await Projects.create(db, { name: 'Second', orgId: org.id })
+    const third = await Projects.create(db, { name: 'Third', orgId: org.id })
+    const { record, token } = await ApiKeys.mintBounded(db, state, {
+      orgId: org.id,
+      projectId: first.id,
+      scopes: ['routes:read'],
+    })
+    const scope = {
+      apiKeyId: record.id,
+      environment: record.environment,
+      orgId: org.id,
+      projectId: first.id,
+    }
+    await RoutesPolicies.save(db, { policy: { maxAmount: '7' }, scope, version: 0 })
+    await ApiKeys.updateBounded(db, state, record.id, { projectId: second.id })
+    expect(await RoutesPolicies.get(db, scope)).toMatchObject({ policy: {}, version: 2 })
+    await Projects.deleteProject(db, first.id)
+
+    await expect(
+      ApiKeys.updateBounded(db, state, record.id, { projectId: third.id }),
+    ).resolves.toMatchObject({ projectId: third.id, scopes: ['routes:read'] })
+    expect(await ApiKeys.resolve(state, token)).toMatchObject({
+      projectId: third.id,
+      scopes: ['routes:read'],
+    })
+    expect(await RoutesPolicies.get(db, scope)).toMatchObject({ policy: {}, version: 3 })
+    expect(await ApiKeyAdmissions.get(db, record.id)).toMatchObject({ projectId: third.id })
+  })
+
+  test('leaves a reassigned key unprivileged when policy clear rolls back at SQL commit', async () => {
+    const db = TestApp.database()
+    const state = Store.memory()
+    const sourceOwner = await Organizations.create(db, { name: 'Source boundary' })
+    const destinationOwner = await Organizations.create(db, { name: 'Destination boundary' })
+    const minted = await ApiKeys.mintBounded(db, state, {
+      orgId: sourceOwner.id,
+      scopes: ['data:read'],
+    })
+    const sourceScope = {
+      apiKeyId: minted.record.id,
+      environment: minted.record.environment,
+      orgId: sourceOwner.id,
+    } as const
+    await RoutesPolicies.save(db, {
+      policy: { maxAmount: '29' },
+      scope: sourceScope,
+      version: 0,
+    })
+    await installDeferredAdmissionFailure(db, destinationOwner.id)
+
+    await expect(
+      ApiKeys.updateBounded(db, state, minted.record.id, { orgId: destinationOwner.id }),
+    ).rejects.toThrow('deferred admission failure')
+
+    const moved = (await ApiKeys.get(state, minted.record.id))!
+    expect(moved).toMatchObject({ orgId: destinationOwner.id, scopes: [] })
+    await expect(ApiKeyAdmissions.matches(db, moved)).resolves.toBe(false)
+    expect(await ApiKeyAdmissions.get(db, minted.record.id)).toMatchObject({
+      orgId: sourceOwner.id,
+      projectId: null,
+    })
+    expect(await RoutesPolicies.get(db, sourceScope)).toMatchObject({
+      policy: { maxAmount: '29' },
+      version: 1,
+    })
+  })
+
   test('rejects an update when the locked key no longer has the expected owner', async () => {
     const db = TestApp.database()
     const state = Store.memory()
@@ -1032,9 +1507,156 @@ describe('revoke', () => {
 
     // The id index entry maps id -> record key; revoke uses it directly.
     expect(await ApiKeys.revoke(state, record.id)).toBe(true)
-    // The record and both index entries are gone.
+    // Only the durable revocation marker remains.
     const { keys } = await state.list({ prefix: 'apikey' })
-    expect(keys).toMatchInlineSnapshot(`[]`)
+    expect(keys.map(({ name }) => name.replace(record.id, '<id>'))).toMatchInlineSnapshot(`
+      [
+        "apikey_revoked:<id>",
+      ]
+    `)
+  })
+})
+
+describe('reconcileOwnerDeletion', () => {
+  test('removes a fence only when SQL deletion did not commit', async () => {
+    const db = TestApp.database()
+    const state = Store.memory()
+    const owner = { orgId: 'org_retained' }
+    await ApiKeys.markOwnerDeleting(state, owner)
+
+    await ApiKeys.reconcileOwnerDeletion(db, state, owner)
+
+    expect(await ApiKeys.isOwnerDeleted(state, owner)).toBe(false)
+    await ApiKeys.markOwnerDeleting(state, owner)
+    await db.transaction((tx) => ApiKeyOwnerTombstones.markDeleted(tx, owner))
+    await ApiKeys.reconcileOwnerDeletion(db, state, owner)
+    expect(await ApiKeys.isOwnerDeleted(state, owner)).toBe(true)
+  })
+
+  test('waits for a concurrent deletion before reconciling its fence', async () => {
+    const database = TestApp.databaseFactory()
+    const state = Store.memory()
+    const owner = { orgId: 'org_concurrent_deletion' }
+    const locked = Promise.withResolvers<void>()
+    const commit = Promise.withResolvers<void>()
+    const deletion = database().transaction(async (tx) => {
+      await ApiKeyOwnerTombstones.markDeleted(tx, owner)
+      await ApiKeys.markOwnerDeleting(state, owner)
+      locked.resolve()
+      await commit.promise
+    })
+    await locked.promise
+
+    const reconciliation = ApiKeys.reconcileOwnerDeletion(database(), state, owner)
+    try {
+      await vi.waitFor(async () => {
+        const result = await sql<{ waiting: boolean }>`
+          SELECT EXISTS (
+            SELECT 1 FROM pg_locks WHERE locktype = 'advisory' AND NOT granted
+            AND objid::bigint = (hashtextextended(${'organization:' + owner.orgId}, 0) & 4294967295)
+          ) AS waiting
+        `.execute(database().kysely)
+        expect(result.rows[0]?.waiting).toBe(true)
+      })
+    } finally {
+      commit.resolve()
+    }
+    await Promise.all([deletion, reconciliation])
+
+    expect(await ApiKeys.isOwnerDeleted(state, owner)).toBe(true)
+  })
+})
+
+describe('isFenced', () => {
+  test('reports revocation and organization or project deletion in one read', async () => {
+    const state = Store.memory()
+    const key = { id: 'key_fenced', orgId: 'org_fenced', projectId: 'prj_fenced' }
+    const sibling = { id: 'key_sibling', orgId: 'org_fenced', projectId: 'prj_sibling' }
+
+    expect(await ApiKeys.isFenced(state, key)).toBe(false)
+
+    await ApiKeys.markOwnerDeleting(state, { orgId: key.orgId, projectId: key.projectId })
+    expect([await ApiKeys.isFenced(state, key), await ApiKeys.isFenced(state, sibling)]).toEqual([
+      true,
+      false,
+    ])
+
+    await ApiKeys.markOwnerDeleting(state, { orgId: key.orgId })
+    expect(await ApiKeys.isFenced(state, sibling)).toBe(true)
+  })
+
+  test('reports a revoked key without an owner fence', async () => {
+    const state = Store.memory()
+    const key = { id: 'key_revoked_only', orgId: 'org_live' }
+
+    await ApiKeys.revoke(state, key.id)
+
+    expect(await ApiKeys.isFenced(state, key)).toBe(true)
+    expect(await ApiKeys.isFenced(state, { ...key, id: 'key_other' })).toBe(false)
+  })
+})
+
+describe('revokeBounded', () => {
+  test.each(['missing', 'stale'])('fences a key with a %s ID index', async (index) => {
+    const db = TestApp.database()
+    const state = Store.memory()
+    const { record, token } = await ApiKeys.mintBounded(db, state, {
+      orgId: 'org_revocation',
+      scopes: ['data:read'],
+    })
+    const recordKey = (await state.get(`apikey_id:${record.id}`))!
+    if (index === 'missing') await state.delete(`apikey_id:${record.id}`)
+    else await state.put(`apikey_id:${record.id}`, 'apikey:missing')
+
+    await expect(ApiKeys.revokeBounded(db, state, record.id)).resolves.toBe(true)
+
+    expect(await state.get(recordKey)).not.toBeNull()
+    expect(await ApiKeys.resolve(state, token)).toBeNull()
+  })
+
+  test('rejects a failed KV fence write and permits retry', async () => {
+    const db = TestApp.database()
+    const backing = Store.memory()
+    const { record, token } = await ApiKeys.mintBounded(db, backing, {
+      orgId: 'org_revocation',
+      scopes: ['data:read'],
+    })
+    await backing.delete(`apikey_id:${record.id}`)
+    const state = Store.from({
+      ...backing,
+      async put(key, value, options) {
+        if (key === `apikey_revoked:${record.id}`) throw new Error('KV write failed')
+        await backing.put(key, value, options)
+      },
+    })
+
+    await expect(ApiKeys.revokeBounded(db, state, record.id)).rejects.toThrow('KV write failed')
+    await expect(ApiKeyAdmissions.exists(db, record.id)).resolves.toBe(true)
+    expect(await ApiKeys.resolve(backing, token)).not.toBeNull()
+
+    await expect(ApiKeys.revokeBounded(db, backing, record.id)).resolves.toBe(true)
+    expect(await ApiKeys.resolve(backing, token)).toBeNull()
+  })
+
+  test('keeps the KV fence when record cleanup fails', async () => {
+    const db = TestApp.database()
+    const backing = Store.memory()
+    const { record, token } = await ApiKeys.mintBounded(db, backing, {
+      orgId: 'org_revocation',
+      scopes: ['data:read'],
+    })
+    const recordKey = (await backing.get(`apikey_id:${record.id}`))!
+    const state = Store.from({
+      ...backing,
+      async delete(key) {
+        if (key === recordKey) throw new Error('KV delete failed')
+        await backing.delete(key)
+      },
+    })
+
+    await expect(ApiKeys.revokeBounded(db, state, record.id)).rejects.toThrow('KV delete failed')
+    expect(await backing.get(recordKey)).not.toBeNull()
+    expect(await ApiKeys.resolve(backing, token)).toBeNull()
   })
 })
 
@@ -1435,3 +2057,25 @@ describe('backfill', () => {
     expect(await ApiKeys.revoke(state, record.id)).toBe(true)
   })
 })
+
+async function installDeferredAdmissionFailure(
+  db: ReturnType<typeof TestApp.database>,
+  orgId: string,
+) {
+  await sql
+    .raw(`
+    CREATE FUNCTION fail_admission_at_commit() RETURNS trigger AS $$
+    BEGIN
+      IF NEW.org_id = '${orgId}' THEN
+        RAISE EXCEPTION 'deferred admission failure';
+      END IF;
+      RETURN NEW;
+    END;
+    $$ LANGUAGE plpgsql;
+    CREATE CONSTRAINT TRIGGER fail_admission_at_commit
+      AFTER INSERT OR UPDATE ON api_key_admissions
+      DEFERRABLE INITIALLY DEFERRED
+      FOR EACH ROW EXECUTE FUNCTION fail_admission_at_commit();
+  `)
+    .execute(db.kysely)
+}