pilothub 0.0.1

package/convex/skillStatEvents.ts

@@ -0,0 +1,568 @@
+/**
+ * Skill Stat Events - Event-sourced stats processing for skills
+ *
+ * Instead of updating skill stats synchronously in the hot path (which can cause
+ * contention when multiple users download/star/install the same skill), we insert
+ * lightweight event records and process them in batches via a cron job.
+ *
+ * Flow:
+ * 1. User action (download, star, install) → insertStatEvent() writes to skillStatEvents table
+ * 2. Cron job runs every 5 minutes → processSkillStatEventsInternal() processes batches
+ * 3. Events are aggregated per-skill to minimize database operations
+ * 4. Stats are applied to skill documents and daily stats tables
+ * 5. Events are marked as processed (kept forever for auditing)
+ */
+
+import { v } from 'convex/values'
+import { internal } from './_generated/api'
+import type { Doc, Id } from './_generated/dataModel'
+import type { MutationCtx } from './_generated/server'
+import { internalAction, internalMutation, internalQuery } from './_generated/server'
+import { applySkillStatDeltas, bumpDailySkillStats } from './lib/skillStats'
+
+/**
+ * Event types that affect skill stats:
+ *
+ * - download: User downloaded skill as zip (+1 downloads)
+ * - star: User starred the skill (+1 stars)
+ * - unstar: User removed their star (-1 stars)
+ * - install_new: First time this user installed this skill (+1 installsAllTime, +1 installsCurrent)
+ * - install_reactivate: User re-added skill after removing it (+1 installsCurrent only)
+ * - install_deactivate: User removed skill from all projects (-1 installsCurrent)
+ * - install_clear: User cleared all telemetry data (custom delta for both allTime and current)
+ */
+export type StatEventKind =
+  | 'download'
+  | 'star'
+  | 'unstar'
+  | 'install_new'
+  | 'install_reactivate'
+  | 'install_deactivate'
+  | 'install_clear'
+
+/**
+ * Insert a stat event to be processed later by the cron job.
+ *
+ * This is called from the hot path (downloads, stars, telemetry) instead of
+ * directly updating skill stats. It's a single insert with no read-modify-write
+ * cycle, so it's fast and doesn't contend with other operations on the same skill.
+ *
+ * @param ctx - Mutation context
+ * @param params.skillId - The skill being affected
+ * @param params.kind - Type of event (download, star, install_new, etc.)
+ * @param params.occurredAt - When the event happened (defaults to now). Important for
+ *                            daily stats bucketing - we want downloads at 11:55 PM Monday
+ *                            to count toward Monday's stats even if processed on Tuesday.
+ * @param params.delta - Only used for install_clear events, specifies exact delta amounts
+ */
+export async function insertStatEvent(
+  ctx: MutationCtx,
+  params: {
+    skillId: Id<'skills'>
+    kind: StatEventKind
+    occurredAt?: number
+    delta?: { allTime: number; current: number }
+  },
+) {
+  await ctx.db.insert('skillStatEvents', {
+    skillId: params.skillId,
+    kind: params.kind,
+    delta: params.delta,
+    occurredAt: params.occurredAt ?? Date.now(),
+    processedAt: undefined,
+  })
+}
+
+/**
+ * Aggregated deltas for a single skill after processing multiple events.
+ *
+ * When we process a batch of 100 events, many might be for the same skill.
+ * Instead of updating the skill document once per event, we aggregate all
+ * events for each skill and apply a single update.
+ *
+ * The downloadEvents and installNewEvents arrays store the original timestamps
+ * so we can update daily stats with the correct day bucket for each event.
+ */
+type AggregatedDeltas = {
+  downloads: number
+  stars: number
+  installsAllTime: number
+  installsCurrent: number
+  /** Original timestamps for each download event (for daily stats bucketing) */
+  downloadEvents: number[]
+  /** Original timestamps for each new install event (for daily stats bucketing) */
+  installNewEvents: number[]
+}
+
+/**
+ * Aggregate multiple events for a single skill into net deltas.
+ *
+ * Example: If a skill has these events in the batch:
+ *   - download (Mon 11pm)
+ *   - download (Tue 1am)
+ *   - star
+ *   - unstar
+ *   - star
+ *
+ * The result would be:
+ *   - downloads: 2
+ *   - stars: 1 (net: +1 -1 +1 = +1)
+ *   - downloadEvents: [<Mon 11pm timestamp>, <Tue 1am timestamp>]
+ *
+ * This aggregation reduces the number of database operations from N events
+ * to 1 skill update + N daily stat updates (which themselves may coalesce
+ * if multiple events fall on the same day).
+ */
+function aggregateEvents(events: Doc<'skillStatEvents'>[]): AggregatedDeltas {
+  const result: AggregatedDeltas = {
+    downloads: 0,
+    stars: 0,
+    installsAllTime: 0,
+    installsCurrent: 0,
+    downloadEvents: [],
+    installNewEvents: [],
+  }
+
+  for (const event of events) {
+    switch (event.kind) {
+      case 'download':
+        result.downloads += 1
+        result.downloadEvents.push(event.occurredAt)
+        break
+      case 'star':
+        result.stars += 1
+        break
+      case 'unstar':
+        result.stars -= 1
+        break
+      case 'install_new':
+        // New user installing for the first time: count toward both lifetime and current
+        result.installsAllTime += 1
+        result.installsCurrent += 1
+        result.installNewEvents.push(event.occurredAt)
+        break
+      case 'install_reactivate':
+        // User re-added skill after removing: only affects current count
+        result.installsCurrent += 1
+        break
+      case 'install_deactivate':
+        // User removed skill from all projects: only affects current count
+        result.installsCurrent -= 1
+        break
+      case 'install_clear':
+        // User cleared telemetry: uses custom delta values (typically negative)
+        if (event.delta) {
+          result.installsAllTime += event.delta.allTime
+          result.installsCurrent += event.delta.current
+        }
+        break
+    }
+  }
+
+  return result
+}
+
+/**
+ * Process a batch of unprocessed stat events.
+ *
+ * Called by cron every 5 minutes. Processes up to batchSize events (default 100).
+ * If the batch is full, schedules an immediate follow-up run to drain the queue.
+ *
+ * Processing steps:
+ * 1. Query unprocessed events (processedAt is undefined)
+ * 2. Group events by skillId to minimize skill document fetches
+ * 3. For each skill:
+ *    a. Fetch the skill document once
+ *    b. Aggregate all events for this skill into net deltas
+ *    c. Apply deltas to skill stats (downloads, stars, installs)
+ *    d. Update daily stats for trending (using original event timestamps)
+ *    e. Mark all events as processed
+ * 4. If batch was full, schedule another run immediately
+ *
+ * Aggregation levels:
+ * - Level 1: Batch of 100 events from the queue
+ * - Level 2: Group by skillId (e.g., 100 events → 30 unique skills)
+ * - Level 3: Aggregate events per skill (e.g., 5 events → 1 skill update)
+ * - Level 4: Daily stats may coalesce (e.g., 3 downloads same day → 1 upsert)
+ */
+export const processSkillStatEventsInternal = internalMutation({
+  args: { batchSize: v.optional(v.number()) },
+  handler: async (ctx, args) => {
+    const batchSize = args.batchSize ?? 100
+    const now = Date.now()
+
+    // Level 1: Fetch a batch of unprocessed events
+    const events = await ctx.db
+      .query('skillStatEvents')
+      .withIndex('by_unprocessed', (q) => q.eq('processedAt', undefined))
+      .take(batchSize)
+
+    if (events.length === 0) {
+      return { processed: 0 }
+    }
+
+    // Level 2: Group events by skillId to minimize database reads
+    // Instead of fetching the same skill document multiple times,
+    // we fetch it once and process all its events together
+    const eventsBySkill = new Map<Id<'skills'>, Doc<'skillStatEvents'>[]>()
+    for (const event of events) {
+      const existing = eventsBySkill.get(event.skillId) ?? []
+      existing.push(event)
+      eventsBySkill.set(event.skillId, existing)
+    }
+
+    // Process each skill's events
+    for (const [skillId, skillEvents] of eventsBySkill) {
+      const skill = await ctx.db.get(skillId)
+
+      // Skill was deleted - just mark events as processed
+      if (!skill) {
+        for (const event of skillEvents) {
+          await ctx.db.patch(event._id, { processedAt: now })
+        }
+        continue
+      }
+
+      // Level 3: Aggregate all events for this skill into net deltas
+      // e.g., 3 downloads + 2 stars - 1 unstar → { downloads: 3, stars: 1 }
+      const deltas = aggregateEvents(skillEvents)
+
+      // Apply aggregated deltas to skill stats (single update per skill)
+      if (
+        deltas.downloads !== 0 ||
+        deltas.stars !== 0 ||
+        deltas.installsAllTime !== 0 ||
+        deltas.installsCurrent !== 0
+      ) {
+        const patch = applySkillStatDeltas(skill, {
+          downloads: deltas.downloads,
+          stars: deltas.stars,
+          installsAllTime: deltas.installsAllTime,
+          installsCurrent: deltas.installsCurrent,
+        })
+        await ctx.db.patch(skill._id, {
+          ...patch,
+          updatedAt: now,
+        })
+      }
+
+      // Update daily stats for trending/leaderboards
+      // We use the ORIGINAL event timestamp (occurredAt) so that:
+      // - A download at Mon 11:55 PM counts toward Monday's stats
+      // - Even if the cron processes it on Tuesday
+      //
+      // Level 4: bumpDailySkillStats does its own coalescing - multiple
+      // events on the same day will update the same daily record
+      for (const occurredAt of deltas.downloadEvents) {
+        await bumpDailySkillStats(ctx, { skillId, now: occurredAt, downloads: 1 })
+      }
+      for (const occurredAt of deltas.installNewEvents) {
+        await bumpDailySkillStats(ctx, { skillId, now: occurredAt, installs: 1 })
+      }
+
+      // Mark all events for this skill as processed
+      for (const event of skillEvents) {
+        await ctx.db.patch(event._id, { processedAt: now })
+      }
+    }
+
+    // If we hit the batch limit, there may be more events waiting.
+    // Schedule an immediate follow-up run to drain the queue.
+    // This ensures high-volume periods don't create a backlog.
+    if (events.length === batchSize) {
+      await ctx.scheduler.runAfter(0, internal.skillStatEvents.processSkillStatEventsInternal, {
+        batchSize,
+      })
+    }
+
+    return { processed: events.length }
+  },
+})
+
+// ============================================================================
+// Action-based processing (cursor-based, runs outside transaction window)
+// ============================================================================
+
+const CURSOR_KEY = 'skill_stat_events'
+const EVENT_BATCH_SIZE = 500
+const MAX_SKILLS_PER_RUN = 500
+
+/**
+ * Fetch a batch of events after the given cursor (by _creationTime).
+ * Returns events sorted by _creationTime ascending.
+ */
+export const getUnprocessedEventBatch = internalQuery({
+  args: {
+    cursorCreationTime: v.optional(v.number()),
+    limit: v.optional(v.number()),
+  },
+  handler: async (ctx, args) => {
+    const limit = args.limit ?? EVENT_BATCH_SIZE
+    const cursor = args.cursorCreationTime
+
+    // Query events after the cursor using the built-in creation time index
+    const events = await ctx.db
+      .query('skillStatEvents')
+      .withIndex('by_creation_time', (q) =>
+        cursor !== undefined ? q.gt('_creationTime', cursor) : q,
+      )
+      .take(limit)
+    return events
+  },
+})
+
+/**
+ * Get the current cursor position from the cursors table.
+ */
+export const getStatEventCursor = internalQuery({
+  args: {},
+  handler: async (ctx) => {
+    const cursor = await ctx.db
+      .query('skillStatUpdateCursors')
+      .withIndex('by_key', (q) => q.eq('key', CURSOR_KEY))
+      .unique()
+    return cursor?.cursorCreationTime
+  },
+})
+
+/**
+ * Validator for skill deltas passed to the mutation.
+ */
+const skillDeltaValidator = v.object({
+  skillId: v.id('skills'),
+  downloads: v.number(),
+  stars: v.number(),
+  installsAllTime: v.number(),
+  installsCurrent: v.number(),
+  downloadEvents: v.array(v.number()),
+  installNewEvents: v.array(v.number()),
+})
+
+/**
+ * Apply aggregated stats to skills and update the cursor.
+ * This is a single atomic mutation that:
+ * 1. Updates all affected skills with their aggregated deltas
+ * 2. Updates daily stats for trending
+ * 3. Advances the cursor to the new position
+ */
+export const applyAggregatedStatsAndUpdateCursor = internalMutation({
+  args: {
+    skillDeltas: v.array(skillDeltaValidator),
+    newCursor: v.number(),
+  },
+  handler: async (ctx, args) => {
+    const now = Date.now()
+
+    // Process each skill's aggregated deltas
+    for (const delta of args.skillDeltas) {
+      const skill = await ctx.db.get(delta.skillId)
+
+      // Skill was deleted - skip
+      if (!skill) {
+        continue
+      }
+
+      // Apply aggregated deltas to skill stats
+      if (
+        delta.downloads !== 0 ||
+        delta.stars !== 0 ||
+        delta.installsAllTime !== 0 ||
+        delta.installsCurrent !== 0
+      ) {
+        const patch = applySkillStatDeltas(skill, {
+          downloads: delta.downloads,
+          stars: delta.stars,
+          installsAllTime: delta.installsAllTime,
+          installsCurrent: delta.installsCurrent,
+        })
+        await ctx.db.patch(skill._id, {
+          ...patch,
+          updatedAt: now,
+        })
+      }
+
+      // Update daily stats for trending/leaderboards
+      for (const occurredAt of delta.downloadEvents) {
+        await bumpDailySkillStats(ctx, { skillId: delta.skillId, now: occurredAt, downloads: 1 })
+      }
+      for (const occurredAt of delta.installNewEvents) {
+        await bumpDailySkillStats(ctx, { skillId: delta.skillId, now: occurredAt, installs: 1 })
+      }
+    }
+
+    // Update cursor position (upsert)
+    const existingCursor = await ctx.db
+      .query('skillStatUpdateCursors')
+      .withIndex('by_key', (q) => q.eq('key', CURSOR_KEY))
+      .unique()
+
+    if (existingCursor) {
+      await ctx.db.patch(existingCursor._id, {
+        cursorCreationTime: args.newCursor,
+        updatedAt: now,
+      })
+    } else {
+      await ctx.db.insert('skillStatUpdateCursors', {
+        key: CURSOR_KEY,
+        cursorCreationTime: args.newCursor,
+        updatedAt: now,
+      })
+    }
+
+    return { skillsUpdated: args.skillDeltas.length }
+  },
+})
+
+/**
+ * Action that processes skill stat events in batches outside the transaction window.
+ *
+ * Algorithm:
+ * 1. Get current cursor position
+ * 2. Fetch events in batches of 500, aggregating as we go
+ * 3. Stop when we have >= 500 unique skills OR run out of events
+ * 4. Call mutation to apply all deltas and update cursor atomically
+ * 5. Self-schedule if we stopped due to skill limit (not exhaustion)
+ */
+export const processSkillStatEventsAction = internalAction({
+  args: {},
+  handler: async (ctx) => {
+    // Get current cursor position (convert null to undefined for consistency)
+    const cursorResult = await ctx.runQuery(internal.skillStatEvents.getStatEventCursor)
+    let cursor: number | undefined = cursorResult ?? undefined
+
+    console.log(`[STAT-AGG] Starting aggregation, cursor=${cursor ?? 'none'}`)
+
+    // Aggregated deltas per skill
+    const aggregatedBySkill = new Map<
+      Id<'skills'>,
+      {
+        downloads: number
+        stars: number
+        installsAllTime: number
+        installsCurrent: number
+        downloadEvents: number[]
+        installNewEvents: number[]
+      }
+    >()
+
+    let maxCreationTime: number | undefined = cursor
+    let exhausted = false
+    let totalEventsFetched = 0
+
+    // Fetch and aggregate until we have enough skills or run out of events
+    while (aggregatedBySkill.size < MAX_SKILLS_PER_RUN) {
+      const events = await ctx.runQuery(internal.skillStatEvents.getUnprocessedEventBatch, {
+        cursorCreationTime: cursor,
+        limit: EVENT_BATCH_SIZE,
+      })
+
+      if (events.length === 0) {
+        exhausted = true
+        break
+      }
+
+      totalEventsFetched += events.length
+      const skillsBefore = aggregatedBySkill.size
+
+      // Aggregate events into per-skill deltas
+      for (const event of events) {
+        let skillDelta = aggregatedBySkill.get(event.skillId)
+        if (!skillDelta) {
+          skillDelta = {
+            downloads: 0,
+            stars: 0,
+            installsAllTime: 0,
+            installsCurrent: 0,
+            downloadEvents: [],
+            installNewEvents: [],
+          }
+          aggregatedBySkill.set(event.skillId, skillDelta)
+        }
+
+        // Apply event to aggregated deltas
+        switch (event.kind) {
+          case 'download':
+            skillDelta.downloads += 1
+            skillDelta.downloadEvents.push(event.occurredAt)
+            break
+          case 'star':
+            skillDelta.stars += 1
+            break
+          case 'unstar':
+            skillDelta.stars -= 1
+            break
+          case 'install_new':
+            skillDelta.installsAllTime += 1
+            skillDelta.installsCurrent += 1
+            skillDelta.installNewEvents.push(event.occurredAt)
+            break
+          case 'install_reactivate':
+            skillDelta.installsCurrent += 1
+            break
+          case 'install_deactivate':
+            skillDelta.installsCurrent -= 1
+            break
+          case 'install_clear':
+            if (event.delta) {
+              skillDelta.installsAllTime += event.delta.allTime
+              skillDelta.installsCurrent += event.delta.current
+            }
+            break
+        }
+
+        // Track highest _creationTime seen
+        if (maxCreationTime === undefined || event._creationTime > maxCreationTime) {
+          maxCreationTime = event._creationTime
+        }
+      }
+
+      // Update cursor for next batch fetch
+      cursor = events[events.length - 1]._creationTime
+
+      console.log(
+        `[STAT-AGG] Fetched ${events.length} events, ${aggregatedBySkill.size - skillsBefore} new skills (${aggregatedBySkill.size} total)`,
+      )
+
+      // If we got fewer than requested, we've exhausted the events
+      if (events.length < EVENT_BATCH_SIZE) {
+        exhausted = true
+        break
+      }
+    }
+
+    // If we have nothing to process, we're done
+    if (aggregatedBySkill.size === 0 || maxCreationTime === undefined) {
+      console.log('[STAT-AGG] No events to process, done')
+      return { processed: 0, skillsUpdated: 0, exhausted: true }
+    }
+
+    // Convert map to array for mutation
+    const skillDeltas = Array.from(aggregatedBySkill.entries()).map(([skillId, delta]) => ({
+      skillId,
+      ...delta,
+    }))
+
+    console.log(
+      `[STAT-AGG] Running mutation for ${skillDeltas.length} skills (${totalEventsFetched} total events)`,
+    )
+
+    // Apply all deltas and update cursor atomically
+    await ctx.runMutation(internal.skillStatEvents.applyAggregatedStatsAndUpdateCursor, {
+      skillDeltas,
+      newCursor: maxCreationTime,
+    })
+
+    // Self-schedule if we stopped because of skill limit, not exhaustion
+    if (!exhausted) {
+      console.log('[STAT-AGG] More events remaining, self-scheduling')
+      await ctx.scheduler.runAfter(0, internal.skillStatEvents.processSkillStatEventsAction, {})
+    } else {
+      console.log('[STAT-AGG] All events processed, done')
+    }
+
+    return {
+      skillsUpdated: skillDeltas.length,
+      exhausted,
+    }
+  },
+})