tapimo 0.15.1 → 0.18.0

package/src/analytics/Analytics.test.ts

@@ -156,9 +156,9 @@ describe('clickhouse', () => {
   })
 
   test('migrate applies every migration when guards are unsatisfied', async () => {
-    // Guards report their columns missing, so every migration's DDL applies.
+    // Guards report missing schema or backfills, so every migration's DDL applies.
     const fetch = vi.fn<typeof globalThis.fetch>().mockImplementation(async (_url, init) => {
-      if (String(init?.body).includes('system.columns'))
+      if (String(init?.body).includes('SELECT toUInt8'))
         return new Response(JSON.stringify({ data: [{ applied: 0 }] }), { status: 200 })
       return new Response('', { status: 200 })
     })
@@ -173,10 +173,9 @@ describe('clickhouse', () => {
   })
 
   test('migrate skips a guarded migration when its guard is satisfied', async () => {
-    // Guards report their columns present (fresh DB from the baseline CREATE),
-    // so the guarded ALTER is skipped, sparing a scoped user the ALTER grant.
+    // Satisfied guards skip DDL and completed backfills.
     const fetch = vi.fn<typeof globalThis.fetch>().mockImplementation(async (_url, init) => {
-      if (String(init?.body).includes('system.columns'))
+      if (String(init?.body).includes('SELECT toUInt8'))
         return new Response(JSON.stringify({ data: [{ applied: 1 }] }), { status: 200 })
       return new Response('', { status: 200 })
     })
@@ -616,7 +615,6 @@ describe('readProjectUsage', () => {
     expect(query).toContain(`project_id = 'prj_1'`)
     expect(query).toContain(`key_environment = 'sandbox'`)
     expect(query).not.toContain('key_id IN')
-    expect(query).toContain('LIMIT 100')
     expect(query).toContain('status >= 400 OR rpc_error_count > 0')
     expect(new URL(String(fetch.mock.calls[0]![0])).searchParams.get('query')).toBeNull()
     expect(usage).toMatchObject({
@@ -722,50 +720,6 @@ describe('readProjectUsage', () => {
       else expect(query).not.toContain('billing_active = 1')
     },
   )
-
-  test('maps grouped rows into the usage response', async () => {
-    const fetch = vi.fn<typeof globalThis.fetch>().mockImplementation(async (_url, init) => {
-      const sql = String(init?.body)
-      const data = (() => {
-        if (sql.includes('GROUP BY key_id'))
-          return [{ averageDurationMs: '12.3', errors: '1', key_id: 'key_1', requests: '10' }]
-        if (sql.includes('GROUP BY route'))
-          return [{ averageDurationMs: 12.3, errors: 1, requests: 10, route: '/v1/tokens' }]
-        if (sql.includes('GROUP BY status'))
-          return [
-            { requests: 9, status: 200 },
-            { requests: '1', status: '500' },
-          ]
-        if (sql.includes('formatDateTime'))
-          return [{ errors: 1, requests: 10, time: '2026-01-01T00:00:00.000Z' }]
-        return [{ averageDurationMs: '12.3', errors: '1', requests: '10' }]
-      })()
-      return new Response(JSON.stringify({ data }), { status: 200 })
-    })
-    vi.stubGlobal('fetch', fetch)
-
-    const usage = await RequestEvents.readProjectUsage(Analytics.clickhouse(clickHouse), {
-      attributions: [{ projectId: 'prj_1' }],
-      from: '2026-01-01T00:00:00.000Z',
-      interval: 'day',
-      orgId: 'org_1',
-      to: '2026-01-02T00:00:00.000Z',
-    })
-
-    expect(usage).toEqual({
-      byKey: [{ apiKeyId: 'key_1', averageDurationMs: 12.3, errors: 1, requests: 10 }],
-      byRoute: [{ averageDurationMs: 12.3, errors: 1, requests: 10, route: '/v1/tokens' }],
-      byStatus: [
-        { requests: 9, status: 200 },
-        { requests: 1, status: 500 },
-      ],
-      from: '2026-01-01T00:00:00.000Z',
-      interval: 'day',
-      series: [{ errors: 1, requests: 10, time: '2026-01-01T00:00:00.000Z' }],
-      to: '2026-01-02T00:00:00.000Z',
-      totals: { averageDurationMs: 12.3, errors: 1, requests: 10 },
-    })
-  })
 })
 
 describe('readLastUsedAt', () => {

package/src/analytics/Analytics.ts

@@ -141,7 +141,17 @@ export const migrations: readonly Migration[] = [
         rpc_error_count UInt16 DEFAULT 0,
         request_id Nullable(String),
         billing_active Nullable(UInt8),
-        timings Map(String, Float64)
+        timings Map(String, Float64),
+        PROJECTION request_usage_by_org (
+          SELECT duration_ms, key_environment, key_id, org_id, principal_type,
+            project_id, route, rpc_error_count, status, timestamp
+          ORDER BY (ifNull(org_id, ''), ifNull(project_id, ''), timestamp)
+        ),
+        PROJECTION request_usage_by_key (
+          SELECT duration_ms, key_environment, key_id, org_id, principal_type,
+            project_id, route, rpc_error_count, status, timestamp
+          ORDER BY (ifNull(key_id, ''), timestamp)
+        )
       )
       ENGINE = MergeTree
       ORDER BY (timestamp)
@@ -447,6 +457,60 @@ export const migrations: readonly Migration[] = [
       ORDER BY (service, environment, routes_environment, stage, provider, method, resource_id)
     `,
   },
+  {
+    name: '0013_request_usage_org_projection',
+    guard: `
+      SELECT toUInt8(count() = 1) FROM system.projections
+      WHERE database = currentDatabase() AND table = 'request_events'
+        AND name = 'request_usage_by_org'
+    `,
+    sql: `
+      ALTER TABLE request_events ADD PROJECTION IF NOT EXISTS request_usage_by_org (
+        SELECT duration_ms, key_environment, key_id, org_id, principal_type,
+          project_id, route, rpc_error_count, status, timestamp
+        ORDER BY (ifNull(org_id, ''), ifNull(project_id, ''), timestamp)
+      )
+    `,
+  },
+  {
+    name: '0014_request_usage_key_projection',
+    guard: `
+      SELECT toUInt8(count() = 1) FROM system.projections
+      WHERE database = currentDatabase() AND table = 'request_events'
+        AND name = 'request_usage_by_key'
+    `,
+    sql: `
+      ALTER TABLE request_events ADD PROJECTION IF NOT EXISTS request_usage_by_key (
+        SELECT duration_ms, key_environment, key_id, org_id, principal_type,
+          project_id, route, rpc_error_count, status, timestamp
+        ORDER BY (ifNull(key_id, ''), timestamp)
+      )
+    `,
+  },
+  {
+    name: '0015_request_usage_org_backfill',
+    guard: `
+      SELECT toUInt8(count() = 0) FROM system.parts
+      WHERE database = currentDatabase() AND table = 'request_events' AND active
+        AND NOT has(projections, 'request_usage_by_org')
+    `,
+    sql: `
+      ALTER TABLE request_events MATERIALIZE PROJECTION request_usage_by_org
+        SETTINGS mutations_sync = 1
+    `,
+  },
+  {
+    name: '0016_request_usage_key_backfill',
+    guard: `
+      SELECT toUInt8(count() = 0) FROM system.parts
+      WHERE database = currentDatabase() AND table = 'request_events' AND active
+        AND NOT has(projections, 'request_usage_by_key')
+    `,
+    sql: `
+      ALTER TABLE request_events MATERIALIZE PROJECTION request_usage_by_key
+        SETTINGS mutations_sync = 1
+    `,
+  },
 ]
 
 /**

package/src/analytics/tables/requestEvents.test.ts

@@ -71,6 +71,12 @@ describe('readBillableCounts', () => {
           request_id: 'request-payment-unavailable',
           status: 429,
         },
+        {
+          ...rows[0]!,
+          error_code: 'client_canceled',
+          request_id: 'request-client-canceled',
+          status: 499,
+        },
       )
 
       const options = {
@@ -89,7 +95,7 @@ describe('readBillableCounts', () => {
         .toMatchInlineSnapshot(`
           [
             {
-              "count": 12,
+              "count": 14,
             },
           ]
         `)
@@ -98,3 +104,398 @@ describe('readBillableCounts', () => {
     }
   })
 })
+
+describe('readProjectUsage', () => {
+  let stack: Awaited<ReturnType<typeof Containers.start>>
+  let analytics: Analytics.Analytics
+
+  async function execute(sql: string) {
+    const response = await fetch(
+      `http://${stack.endpoints.clickhouse.host}:${stack.endpoints.clickhouse.port}/?database=tempo_api`,
+      {
+        body: sql,
+        headers: { Authorization: `Basic ${btoa('clickhouse:clickhouse')}` },
+        method: 'POST',
+      },
+    )
+    if (!response.ok) throw new Error(await response.text())
+  }
+
+  beforeAll(async () => {
+    stack = await Containers.start()
+    analytics = Analytics.clickhouse({
+      database: 'tempo_api',
+      password: 'clickhouse',
+      url: `http://${stack.endpoints.clickhouse.host}:${stack.endpoints.clickhouse.port}`,
+      user: 'clickhouse',
+    })
+    await execute(Analytics.migrations[0]!.sql)
+    // Existing production tables predate the projections and must still be backfilled.
+    await execute(`ALTER TABLE request_events
+      DROP PROJECTION request_usage_by_org, DROP PROJECTION request_usage_by_key`)
+    await analytics.insert('request_events', [
+      { ...row, org_id: 'org_backfill', project_id: 'prj_backfill' },
+    ])
+    await analytics.migrate()
+  })
+  afterAll(async () => {
+    await stack?.stop()
+  })
+
+  const row = {
+    billing_active: null,
+    chain_id: null,
+    duration_ms: 10,
+    environment: 'production',
+    error_code: null,
+    key_environment: 'production',
+    key_id: 'key_usage',
+    method: 'GET',
+    org_id: 'org_usage',
+    principal_type: 'api_key',
+    project_id: 'prj_usage',
+    query: '',
+    rate_limit_scope: null,
+    request_id: null,
+    route: '/v1/tokens',
+    rpc_error_code: null,
+    rpc_error_count: 0,
+    rpc_error_data_code: null,
+    service: 'api',
+    status: 200,
+    timestamp: '2026-01-01 00:00:00.000',
+    timings: {},
+  } satisfies RequestEvents.Table
+
+  test('preserves every breakdown, legacy attribution, and JSON-RPC errors', async () => {
+    await analytics.insert('request_events', [
+      row,
+      { ...row, duration_ms: 30, rpc_error_count: 1 },
+      { ...row, duration_ms: 50, key_id: null, status: 502 },
+      {
+        ...row,
+        duration_ms: 70,
+        key_environment: null,
+        key_id: 'key_legacy',
+        org_id: null,
+        project_id: null,
+        route: '/v1/tokenlist',
+        timestamp: '2026-01-02 00:00:00.000',
+      },
+      { ...row, org_id: 'org_other' },
+      { ...row, key_environment: 'sandbox' },
+      { ...row, timestamp: '2025-12-31 00:00:00.000' },
+      { ...row, key_id: 'key_other', org_id: null, project_id: null },
+    ])
+    const usage = await RequestEvents.readProjectUsage(analytics, {
+      attributions: [
+        { environment: 'production', projectId: 'prj_usage' },
+        { apiKeyIds: ['key_legacy'], environment: 'production', projectId: null },
+      ],
+      from: '2026-01-01T00:00:00.000Z',
+      interval: 'day',
+      orgId: 'org_usage',
+      to: '2026-01-02T00:00:00.000Z',
+    })
+    expect(usage).toMatchInlineSnapshot(`
+      {
+        "byKey": [
+          {
+            "apiKeyId": "key_usage",
+            "averageDurationMs": 20,
+            "errors": 1,
+            "requests": 2,
+          },
+          {
+            "apiKeyId": "key_legacy",
+            "averageDurationMs": 70,
+            "errors": 0,
+            "requests": 1,
+          },
+        ],
+        "byRoute": [
+          {
+            "averageDurationMs": 30,
+            "errors": 2,
+            "requests": 3,
+            "route": "/v1/tokens",
+          },
+          {
+            "averageDurationMs": 70,
+            "errors": 0,
+            "requests": 1,
+            "route": "/v1/tokenlist",
+          },
+        ],
+        "byStatus": [
+          {
+            "requests": 3,
+            "status": 200,
+          },
+          {
+            "requests": 1,
+            "status": 502,
+          },
+        ],
+        "from": "2026-01-01T00:00:00.000Z",
+        "interval": "day",
+        "series": [
+          {
+            "errors": 2,
+            "requests": 3,
+            "time": "2026-01-01T00:00:00.000Z",
+          },
+          {
+            "errors": 0,
+            "requests": 1,
+            "time": "2026-01-02T00:00:00.000Z",
+          },
+        ],
+        "to": "2026-01-02T00:00:00.000Z",
+        "totals": {
+          "averageDurationMs": 40,
+          "errors": 2,
+          "requests": 4,
+        },
+      }
+    `)
+  })
+
+  test('returns zero totals for an empty window', async () => {
+    expect(
+      await RequestEvents.readProjectUsage(analytics, {
+        attributions: [],
+        from: '2026-01-01T00:00:00.000Z',
+        interval: 'hour',
+        orgId: 'org_empty',
+        to: '2026-01-02T00:00:00.000Z',
+      }),
+    ).toMatchInlineSnapshot(`
+      {
+        "byKey": [],
+        "byRoute": [],
+        "byStatus": [],
+        "from": "2026-01-01T00:00:00.000Z",
+        "interval": "hour",
+        "series": [],
+        "to": "2026-01-02T00:00:00.000Z",
+        "totals": {
+          "averageDurationMs": 0,
+          "errors": 0,
+          "requests": 0,
+        },
+      }
+    `)
+  })
+
+  test('counts overlapping attribution terms once without deduplicating requests', async () => {
+    await analytics.insert('request_events', [
+      { ...row, org_id: 'org_overlap', project_id: null },
+      { ...row, org_id: 'org_overlap', project_id: null },
+      { ...row, key_environment: null, org_id: null, project_id: null },
+      { ...row, org_id: 'org_overlap' },
+      { ...row, org_id: 'org_other_overlap' },
+    ])
+    const usage = await RequestEvents.readProjectUsage(analytics, {
+      attributions: [
+        { environment: 'production' },
+        { environment: 'production', projectId: 'prj_usage' },
+        { apiKeyIds: ['key_usage'], environment: 'production', projectId: null },
+      ],
+      from: '2026-01-01T00:00:00.000Z',
+      interval: 'hour',
+      orgId: 'org_overlap',
+      to: '2026-01-01T00:00:00.000Z',
+    })
+    expect({ byKey: usage.byKey, totals: usage.totals }).toMatchInlineSnapshot(`
+      {
+        "byKey": [
+          {
+            "apiKeyId": "key_usage",
+            "averageDurationMs": 10,
+            "errors": 0,
+            "requests": 4,
+          },
+        ],
+        "totals": {
+          "averageDurationMs": 10,
+          "errors": 0,
+          "requests": 4,
+        },
+      }
+    `)
+  })
+
+  test('backfills existing parts and skips completed backfills on subsequent migrations', async () => {
+    const projections = await analytics.query<{ name: string }>(`
+      SELECT DISTINCT name FROM system.projection_parts
+      WHERE database = currentDatabase() AND table = 'request_events' AND active
+      ORDER BY name
+    `)
+    expect(projections).toMatchInlineSnapshot(`
+      [
+        {
+          "name": "request_usage_by_key",
+        },
+        {
+          "name": "request_usage_by_org",
+        },
+      ]
+    `)
+    await analytics.migrate()
+    expect(
+      (
+        await RequestEvents.readProjectUsage(analytics, {
+          attributions: [{ environment: 'production', projectId: 'prj_backfill' }],
+          from: '2026-01-01T00:00:00.000Z',
+          interval: 'day',
+          orgId: 'org_backfill',
+          to: '2026-01-01T00:00:00.000Z',
+        })
+      ).totals,
+    ).toMatchInlineSnapshot(`
+      {
+        "averageDurationMs": 10,
+        "errors": 0,
+        "requests": 1,
+      }
+    `)
+  })
+
+  test('migrates a fresh database without issuing ALTER statements', async () => {
+    await execute('CREATE DATABASE tempo_api_fresh')
+    const url = `http://${stack.endpoints.clickhouse.host}:${stack.endpoints.clickhouse.port}`
+    const preview = Analytics.clickhouse({
+      database: 'tempo_api_fresh',
+      password: 'clickhouse',
+      url,
+      user: 'clickhouse',
+    })
+    await preview.migrate()
+    await preview.insert('request_events', [row])
+    await preview.migrate()
+    await execute('SYSTEM FLUSH LOGS')
+    const [queries] = await analytics.query<{ count: string }>(`
+      SELECT count() AS count FROM system.query_log
+      WHERE current_database = 'tempo_api_fresh' AND query_kind = 'Alter'
+    `)
+    expect(Number(queries?.count)).toBe(0)
+    expect(
+      await preview.query(`
+        SELECT name FROM system.projections
+        WHERE database = currentDatabase() AND table = 'request_events'
+        ORDER BY name
+      `),
+    ).toMatchInlineSnapshot(`
+      [
+        {
+          "name": "request_usage_by_key",
+        },
+        {
+          "name": "request_usage_by_org",
+        },
+      ]
+    `)
+  })
+
+  test('prunes unrelated traffic from project and legacy key scans', async () => {
+    await execute(`
+      INSERT INTO request_events (
+        timestamp, org_id, project_id, key_id, principal_type, key_environment,
+        route, status, duration_ms
+      )
+      SELECT toDateTime64('2026-02-01', 3),
+        concat('org_', leftPad(toString(number % 10000), 24, '0')),
+        if(number % 3 = 0, NULL, concat('prj_', leftPad(toString(number % 10000), 24, '0'))),
+        concat('key_', leftPad(toString(number % 10000), 24, '0')),
+        'api_key', 'production', '/v1/tokens', 200, 10
+      FROM numbers(500000)
+    `)
+    const usage = await RequestEvents.readProjectUsage(analytics, {
+      attributions: [
+        { environment: 'production', projectId: `prj_${'0'.repeat(23)}7` },
+        {
+          apiKeyIds: [`key_${'0'.repeat(23)}7`],
+          environment: 'production',
+          projectId: null,
+        },
+      ],
+      from: '2026-02-01T00:00:00.000Z',
+      interval: 'day',
+      orgId: `org_${'0'.repeat(23)}7`,
+      to: '2026-02-01T00:00:00.000Z',
+    })
+    expect(usage.totals).toMatchInlineSnapshot(`
+      {
+        "averageDurationMs": 10,
+        "errors": 0,
+        "requests": 50,
+      }
+    `)
+    await execute('SYSTEM FLUSH LOGS')
+    const [query] = await analytics.query<{ query: string; read_rows: string }>(`
+      SELECT query, read_rows FROM system.query_log
+      WHERE type = 'QueryFinish' AND query LIKE '%GROUPING SETS%'
+        AND query LIKE '%org_${'0'.repeat(23)}7%'
+        AND query NOT LIKE '%system.query_log%'
+      ORDER BY event_time_microseconds DESC LIMIT 1
+    `)
+    expect(Number(query?.read_rows)).toBeLessThan(100000)
+    const plan = await analytics.query<{ explain: string }>(
+      `EXPLAIN indexes = 1 ${query!.query.replace(/FORMAT JSON\s*$/, '')}`,
+    )
+    expect(plan.map((line) => line.explain).join('\n')).toContain('request_usage_by_org')
+    expect(plan.map((line) => line.explain).join('\n')).toContain('request_usage_by_key')
+  })
+
+  test('limits keys and routes without truncating statuses or time buckets', async () => {
+    await analytics.insert(
+      'request_events',
+      Array.from({ length: 120 }, (_, index) => ({
+        ...row,
+        key_id: `key_${String(index).padStart(3, '0')}`,
+        org_id: 'org_limits',
+        route: `/route/${String(index).padStart(3, '0')}`,
+        status: 200 + index,
+        timestamp: new Date(Date.UTC(2026, 0, 1) + index * 3_600_000)
+          .toISOString()
+          .replace('T', ' ')
+          .replace('Z', ''),
+      })),
+    )
+    const usage = await RequestEvents.readProjectUsage(analytics, {
+      attributions: [{ environment: 'production' }],
+      from: '2026-01-01T00:00:00.000Z',
+      interval: 'hour',
+      orgId: 'org_limits',
+      to: '2026-01-06T00:00:00.000Z',
+    })
+    expect({
+      firstKey: usage.byKey[0]?.apiKeyId,
+      firstTime: usage.series[0]?.time,
+      keys: usage.byKey.length,
+      lastKey: usage.byKey.at(-1)?.apiKeyId,
+      lastTime: usage.series.at(-1)?.time,
+      routes: usage.byRoute.length,
+      series: usage.series.length,
+      statuses: usage.byStatus.length,
+      totals: usage.totals,
+    }).toMatchInlineSnapshot(`
+      {
+        "firstKey": "key_000",
+        "firstTime": "2026-01-01T00:00:00.000Z",
+        "keys": 100,
+        "lastKey": "key_099",
+        "lastTime": "2026-01-05T23:00:00.000Z",
+        "routes": 100,
+        "series": 120,
+        "statuses": 120,
+        "totals": {
+          "averageDurationMs": 10,
+          "errors": 0,
+          "requests": 120,
+        },
+      }
+    `)
+  })
+})