tapimo 0.2.0 → 0.3.0

package/src/analytics/Analytics.ts

@@ -0,0 +1,318 @@
+import * as RequestEvents from './tables/requestEvents.js'
+
+export { createQueueSink } from './tables/requestEvents.js'
+
+/** One org's deduped billable request count for a UTC hour; the reporter's source row. */
+export type BillableCount = RequestEvents.BillableCount
+
+/** Options for {@link readBillableRequestCounts}. */
+export type BillableCountsOptions = RequestEvents.BillableCountsOptions
+
+/**
+ * Reads deduped billable request counts grouped by `(org, UTC hour)` from
+ * `request_events` — the request-metered billing source. Resolves the source
+ * at the leaf so a Workers factory builds a fresh connection.
+ *
+ * @param source - The analytics store or factory.
+ * @param options - The read options.
+ * @returns One deduped count per `(org, hour)`.
+ */
+export function readBillableRequestCounts(
+  source: Source,
+  options: BillableCountsOptions,
+): Promise<BillableCount[]> {
+  return RequestEvents.readBillableCounts(get(source), options)
+}
+
+/**
+ * ClickHouse schema; table repository modules (`tables/*`) type their rows
+ * through it.
+ */
+export type Database = {
+  request_events: RequestEvents.Table
+}
+
+/**
+ * A ClickHouse-backed analytics store: row inserts, JSON reads, and schema
+ * migrations. Table repository modules (`tables/*`) query through it.
+ */
+export type Analytics = {
+  /** Inserts rows into a table. */
+  insert<name extends keyof Database>(table: name, rows: readonly Database[name][]): Promise<void>
+  /** Applies any pending schema migrations; safe to re-run. */
+  migrate(): Promise<void>
+  /** Runs a read query, returning JSON rows. */
+  query<row>(sql: string): Promise<row[]>
+}
+
+/**
+ * An {@link Analytics}, or a factory for per-request construction. Resolve at
+ * the leaf with {@link get}.
+ */
+export type Source = Analytics | (() => Analytics)
+
+/** One stored request row (`request_events`); hosts use it in their queue-body union. */
+export type Event = RequestEvents.Table
+
+/** Queue name {@link handleQueue} consumes; the host's queue producer/consumer bindings must reference it. */
+export const queueName = 'api-request-analytics'
+
+/**
+ * Resolves a {@link Source} to an {@link Analytics}. Call at the request leaf
+ * so factories create per-request instances; singletons resolve to themselves.
+ *
+ * @param source - The analytics store or factory.
+ * @returns The resolved {@link Analytics}.
+ */
+export function get(source: Source): Analytics {
+  return typeof source === 'function' ? source() : source
+}
+
+/** One schema migration: a single idempotent DDL statement, safe to re-apply. */
+type Migration = {
+  /** Unique, ordered migration name (e.g. `0001_request_events`). */
+  name: string
+  /** Idempotent DDL applied by {@link Analytics.migrate}. */
+  sql: string
+  /**
+   * Optional `SELECT` run before {@link Migration.sql}; a nonzero first value
+   * skips the migration. Lets a scoped user without an `ALTER` grant (preview)
+   * skip an `ALTER` whose columns already exist from the baseline `CREATE`.
+   */
+  guard?: string
+}
+
+/**
+ * Ordered migrations for {@link Analytics.migrate}. ClickHouse lacks
+ * transactional DDL and advisory locks, so each statement is idempotent
+ * (`IF NOT EXISTS`) and re-applied on every run; no applied-migrations ledger.
+ */
+export const migrations: readonly Migration[] = [
+  {
+    name: '0001_request_events',
+    sql: `
+      CREATE TABLE IF NOT EXISTS request_events (
+        timestamp DateTime64(3, 'UTC'),
+        service LowCardinality(String),
+        environment LowCardinality(String),
+        method LowCardinality(String),
+        route LowCardinality(String),
+        status UInt16,
+        duration_ms Float64,
+        principal_type LowCardinality(String),
+        key_id Nullable(String),
+        key_environment Nullable(String),
+        org_id Nullable(String),
+        project_id Nullable(String),
+        rate_limit_scope Nullable(String),
+        error_code Nullable(String),
+        request_id Nullable(String),
+        billing_active Nullable(UInt8)
+      )
+      ENGINE = MergeTree
+      ORDER BY (timestamp)
+    `,
+  },
+  {
+    name: '0002_request_events_billing',
+    // Patches tables created before the billing columns were folded into 0001
+    // (production). Fresh databases already carry them from the CREATE, so the
+    // guard skips this ALTER there, where the scoped migrate user (preview)
+    // lacks the ALTER grant. ADD COLUMN IF NOT EXISTS keeps it idempotent.
+    guard: `
+      SELECT toUInt8(count() = 2) FROM system.columns
+      WHERE database = currentDatabase()
+        AND table = 'request_events'
+        AND name IN ('request_id', 'billing_active')
+    `,
+    sql: `
+      ALTER TABLE request_events
+        ADD COLUMN IF NOT EXISTS request_id Nullable(String),
+        ADD COLUMN IF NOT EXISTS billing_active Nullable(UInt8)
+    `,
+  },
+]
+
+/**
+ * Creates a ClickHouse-backed {@link Analytics} over the HTTP interface.
+ * Requests scope to `database` (the `?database=` parameter); table names are
+ * code-owned by the `tables/*` modules, not configuration.
+ *
+ * @param options - ClickHouse connection options.
+ * @returns The analytics store.
+ */
+export function clickhouse(options: clickhouse.Options): Analytics {
+  const authorization = `Basic ${btoa(`${options.user}:${options.password}`)}`
+
+  function endpoint() {
+    const url = new URL(options.url)
+    url.searchParams.set('database', options.database)
+    url.searchParams.set('date_time_input_format', 'best_effort')
+    return url
+  }
+
+  // Runs a migration guard SELECT; true (skip) when its first value is nonzero.
+  // A failed guard returns false so the migration still runs and reports.
+  async function guarded(sql: string): Promise<boolean> {
+    const response = await fetch(endpoint(), {
+      body: `${sql} FORMAT JSON`,
+      headers: { Authorization: authorization, 'Content-Type': 'text/plain' },
+      method: 'POST',
+    })
+    if (!response.ok) return false
+    try {
+      const body = (await response.json()) as { data?: Array<Record<string, unknown>> }
+      const first = body.data?.[0]
+      return first ? Number(Object.values(first)[0]) > 0 : false
+    } catch {
+      return false
+    }
+  }
+
+  return {
+    async insert(table, rows) {
+      if (rows.length === 0) return
+      const url = endpoint()
+      url.searchParams.set('query', `INSERT INTO ${quote(table)} FORMAT JSONEachRow`)
+      const response = await fetch(url, {
+        body: `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`,
+        headers: { Authorization: authorization, 'Content-Type': 'application/x-ndjson' },
+        method: 'POST',
+      })
+      if (response.ok) return
+      throw new InsertError(response.status, await response.text())
+    },
+    async migrate() {
+      for (const migration of migrations) {
+        if (migration.guard && (await guarded(migration.guard))) continue
+        const response = await fetch(endpoint(), {
+          body: migration.sql,
+          headers: { Authorization: authorization, 'Content-Type': 'text/plain' },
+          method: 'POST',
+        })
+        if (response.ok) continue
+        throw new MigrateError(migration.name, response.status, await response.text())
+      }
+    },
+    async query<row>(sql: string): Promise<row[]> {
+      const response = await fetch(endpoint(), {
+        body: `${sql} FORMAT JSON`,
+        headers: { Authorization: authorization, 'Content-Type': 'text/plain' },
+        method: 'POST',
+      })
+      if (!response.ok) throw new QueryError(response.status, await response.text())
+      const body = (await response.json()) as { data?: row[] | undefined }
+      return body.data ?? []
+    },
+  }
+}
+
+export declare namespace clickhouse {
+  /** ClickHouse connection options. */
+  type Options = {
+    /** Database the connection scopes to (e.g. `tempo_api`). */
+    database: string
+    /** Password for `user`. */
+    password: string
+    /** ClickHouse HTTPS endpoint. */
+    url: string
+    /** ClickHouse user; ingest and read deployments differ only here. */
+    user: string
+  }
+}
+
+/**
+ * Consumes one queue batch when it is the analytics queue ({@link queueName}):
+ * batch-inserts the queued rows, acking on success and retrying on failure.
+ * Returns whether the batch was handled, so a multi-queue worker can chain:
+ *
+ * ```ts
+ * async queue(batch) {
+ *   if (await Analytics.handleQueue(analytics, batch)) return
+ *   // … other queues …
+ * }
+ * ```
+ *
+ * @param source - The analytics store or factory.
+ * @param batch - The queue batch.
+ * @param options - Options.
+ * @returns Whether the batch belonged to the analytics queue.
+ */
+export async function handleQueue(
+  source: Source,
+  batch: handleQueue.Batch,
+  options: handleQueue.Options = {},
+): Promise<boolean> {
+  if (batch.queue !== (options.queue ?? queueName)) return false
+  await RequestEvents.insertMessages(
+    get(source),
+    batch.messages as readonly RequestEvents.insertMessages.Message[],
+  )
+  return true
+}
+
+export declare namespace handleQueue {
+  /** Options for {@link handleQueue}. */
+  type Options = {
+    /** Queue name to match instead of {@link queueName} (e.g. a per-preview queue). */
+    queue?: string | undefined
+  }
+
+  /** Minimal queue batch shape; Cloudflare's `MessageBatch` satisfies it. */
+  type Batch = {
+    /** Queued messages; bodies are {@link Event} rows when `queue` matches. */
+    messages: readonly Message[]
+    /** Originating queue name. */
+    queue: string
+  }
+
+  /** One queued message. */
+  type Message = {
+    /** Acknowledges the message. */
+    ack(): void
+    /** Queued body; an {@link Event} when the batch is the analytics queue. */
+    body: unknown
+    /** Marks the message for redelivery. */
+    retry(): void
+  }
+}
+
+/** Valid unquoted ClickHouse identifier. */
+const identifier = /^[A-Za-z_][A-Za-z0-9_]*$/
+
+function quote(value: string) {
+  if (identifier.test(value)) return `\`${value}\``
+  throw new ConfigError(value)
+}
+
+/** Rejected ClickHouse identifier. */
+export class ConfigError extends Error {
+  constructor(value: string) {
+    super(`Invalid ClickHouse identifier: ${value}`)
+    this.name = 'Analytics.ConfigError'
+  }
+}
+
+/** Failed ClickHouse insert (non-2xx HTTP response). */
+export class InsertError extends Error {
+  constructor(status: number, body: string) {
+    super(`ClickHouse insert failed with status ${status}: ${body.slice(0, 500)}`)
+    this.name = 'Analytics.InsertError'
+  }
+}
+
+/** Failed ClickHouse schema migration (non-2xx HTTP response). */
+export class MigrateError extends Error {
+  constructor(name: string, status: number, body: string) {
+    super(`ClickHouse migration "${name}" failed with status ${status}: ${body.slice(0, 500)}`)
+    this.name = 'Analytics.MigrateError'
+  }
+}
+
+/** Failed ClickHouse read query (non-2xx HTTP response). */
+export class QueryError extends Error {
+  constructor(status: number, body: string) {
+    super(`ClickHouse query failed with status ${status}: ${body.slice(0, 500)}`)
+    this.name = 'Analytics.QueryError'
+  }
+}