pilothub 0.0.1

package/convex/search.ts

@@ -0,0 +1,254 @@
+import { v } from 'convex/values'
+import { internal } from './_generated/api'
+import type { Doc, Id } from './_generated/dataModel'
+import { action, internalQuery } from './_generated/server'
+import { getSkillBadgeMaps, isSkillHighlighted, type SkillBadgeMap } from './lib/badges'
+import { generateEmbedding } from './lib/embeddings'
+import { toPublicSkill, toPublicSoul } from './lib/public'
+import { matchesExactTokens, tokenize } from './lib/searchText'
+
+type HydratedEntry = {
+  embeddingId: Id<'skillEmbeddings'>
+  skill: NonNullable<ReturnType<typeof toPublicSkill>>
+  version: Doc<'skillVersions'> | null
+  ownerHandle: string | null
+}
+
+type SearchResult = HydratedEntry & { score: number }
+
+function getNextCandidateLimit(current: number, max: number) {
+  const next = Math.min(current * 2, max)
+  return next > current ? next : null
+}
+
+export const searchSkills: ReturnType<typeof action> = action({
+  args: {
+    query: v.string(),
+    limit: v.optional(v.number()),
+    highlightedOnly: v.optional(v.boolean()),
+  },
+  handler: async (ctx, args): Promise<SearchResult[]> => {
+    const query = args.query.trim()
+    if (!query) return []
+    const queryTokens = tokenize(query)
+    if (queryTokens.length === 0) return []
+    let vector: number[]
+    try {
+      vector = await generateEmbedding(query)
+    } catch (error) {
+      console.warn('Search embedding generation failed', error)
+      return []
+    }
+    const limit = args.limit ?? 10
+    // Convex vectorSearch max limit is 256; clamp candidate sizes accordingly.
+    const maxCandidate = Math.min(Math.max(limit * 10, 200), 256)
+    let candidateLimit = Math.min(Math.max(limit * 3, 50), 256)
+    let hydrated: HydratedEntry[] = []
+    let scoreById = new Map<Id<'skillEmbeddings'>, number>()
+    let exactMatches: HydratedEntry[] = []
+
+    while (candidateLimit <= maxCandidate) {
+      const results = await ctx.vectorSearch('skillEmbeddings', 'by_embedding', {
+        vector,
+        limit: candidateLimit,
+        filter: (q) => q.or(q.eq('visibility', 'latest'), q.eq('visibility', 'latest-approved')),
+      })
+
+      hydrated = (await ctx.runQuery(internal.search.hydrateResults, {
+        embeddingIds: results.map((result) => result._id),
+      })) as HydratedEntry[]
+
+      scoreById = new Map<Id<'skillEmbeddings'>, number>(
+        results.map((result) => [result._id, result._score]),
+      )
+
+      const badgeMapEntries = (await ctx.runQuery(internal.search.getSkillBadgeMapsInternal, {
+        skillIds: hydrated.map((entry) => entry.skill._id),
+      })) as Array<[Id<'skills'>, SkillBadgeMap]>
+      const badgeMapBySkillId = new Map(badgeMapEntries)
+      const hydratedWithBadges = hydrated.map((entry) => ({
+        ...entry,
+        skill: {
+          ...entry.skill,
+          badges: badgeMapBySkillId.get(entry.skill._id) ?? {},
+        },
+      }))
+
+      const filtered = args.highlightedOnly
+        ? hydratedWithBadges.filter((entry) => isSkillHighlighted(entry.skill))
+        : hydratedWithBadges
+
+      exactMatches = filtered.filter((entry) =>
+        matchesExactTokens(queryTokens, [
+          entry.skill.displayName,
+          entry.skill.slug,
+          entry.skill.summary,
+        ]),
+      )
+
+      if (exactMatches.length >= limit || results.length < candidateLimit) {
+        break
+      }
+
+      const nextLimit = getNextCandidateLimit(candidateLimit, maxCandidate)
+      if (!nextLimit) break
+      candidateLimit = nextLimit
+    }
+
+    return exactMatches
+      .map((entry) => ({
+        ...entry,
+        score: scoreById.get(entry.embeddingId) ?? 0,
+      }))
+      .filter((entry) => entry.skill)
+      .slice(0, limit)
+  },
+})
+
+export const getBadgeMapsForSkills = internalQuery({
+  args: { skillIds: v.array(v.id('skills')) },
+  handler: async (ctx, args): Promise<Array<[Id<'skills'>, SkillBadgeMap]>> => {
+    const badgeMap = await getSkillBadgeMaps(ctx, args.skillIds)
+    return Array.from(badgeMap.entries())
+  },
+})
+
+export const hydrateResults = internalQuery({
+  args: { embeddingIds: v.array(v.id('skillEmbeddings')) },
+  handler: async (ctx, args): Promise<HydratedEntry[]> => {
+    const ownerHandleCache = new Map<Id<'users'>, Promise<string | null>>()
+
+    const getOwnerHandle = (ownerUserId: Id<'users'>) => {
+      const cached = ownerHandleCache.get(ownerUserId)
+      if (cached) return cached
+      const handlePromise = ctx.db
+        .get(ownerUserId)
+        .then((owner) => owner?.handle ?? owner?._id ?? null)
+      ownerHandleCache.set(ownerUserId, handlePromise)
+      return handlePromise
+    }
+
+    const entries = await Promise.all(
+      args.embeddingIds.map(async (embeddingId) => {
+        const embedding = await ctx.db.get(embeddingId)
+        if (!embedding) return null
+        const skill = await ctx.db.get(embedding.skillId)
+        if (!skill || skill.softDeletedAt) return null
+        const [version, ownerHandle] = await Promise.all([
+          ctx.db.get(embedding.versionId),
+          getOwnerHandle(skill.ownerUserId),
+        ])
+        const publicSkill = toPublicSkill(skill)
+        if (!publicSkill) return null
+        return { embeddingId, skill: publicSkill, version, ownerHandle }
+      }),
+    )
+
+    return entries.filter((entry): entry is HydratedEntry => entry !== null)
+  },
+})
+
+type HydratedSoulEntry = {
+  embeddingId: Id<'soulEmbeddings'>
+  soul: NonNullable<ReturnType<typeof toPublicSoul>>
+  version: Doc<'soulVersions'> | null
+}
+
+type SoulSearchResult = HydratedSoulEntry & { score: number }
+
+export const searchSouls: ReturnType<typeof action> = action({
+  args: {
+    query: v.string(),
+    limit: v.optional(v.number()),
+  },
+  handler: async (ctx, args): Promise<SoulSearchResult[]> => {
+    const query = args.query.trim()
+    if (!query) return []
+    const queryTokens = tokenize(query)
+    if (queryTokens.length === 0) return []
+    let vector: number[]
+    try {
+      vector = await generateEmbedding(query)
+    } catch (error) {
+      console.warn('Search embedding generation failed', error)
+      return []
+    }
+    const limit = args.limit ?? 10
+    // Convex vectorSearch max limit is 256; clamp candidate sizes accordingly.
+    const maxCandidate = Math.min(Math.max(limit * 10, 200), 256)
+    let candidateLimit = Math.min(Math.max(limit * 3, 50), 256)
+    let hydrated: HydratedSoulEntry[] = []
+    let scoreById = new Map<Id<'soulEmbeddings'>, number>()
+    let exactMatches: HydratedSoulEntry[] = []
+
+    while (candidateLimit <= maxCandidate) {
+      const results = await ctx.vectorSearch('soulEmbeddings', 'by_embedding', {
+        vector,
+        limit: candidateLimit,
+        filter: (q) => q.or(q.eq('visibility', 'latest'), q.eq('visibility', 'latest-approved')),
+      })
+
+      hydrated = (await ctx.runQuery(internal.search.hydrateSoulResults, {
+        embeddingIds: results.map((result) => result._id),
+      })) as HydratedSoulEntry[]
+
+      scoreById = new Map<Id<'soulEmbeddings'>, number>(
+        results.map((result) => [result._id, result._score]),
+      )
+
+      exactMatches = hydrated.filter((entry) =>
+        matchesExactTokens(queryTokens, [
+          entry.soul.displayName,
+          entry.soul.slug,
+          entry.soul.summary,
+        ]),
+      )
+
+      if (exactMatches.length >= limit || results.length < candidateLimit) {
+        break
+      }
+
+      const nextLimit = getNextCandidateLimit(candidateLimit, maxCandidate)
+      if (!nextLimit) break
+      candidateLimit = nextLimit
+    }
+
+    return exactMatches
+      .map((entry) => ({
+        ...entry,
+        score: scoreById.get(entry.embeddingId) ?? 0,
+      }))
+      .filter((entry) => entry.soul)
+      .slice(0, limit)
+  },
+})
+
+export const hydrateSoulResults = internalQuery({
+  args: { embeddingIds: v.array(v.id('soulEmbeddings')) },
+  handler: async (ctx, args): Promise<HydratedSoulEntry[]> => {
+    const entries: HydratedSoulEntry[] = []
+
+    for (const embeddingId of args.embeddingIds) {
+      const embedding = await ctx.db.get(embeddingId)
+      if (!embedding) continue
+      const soul = await ctx.db.get(embedding.soulId)
+      if (soul?.softDeletedAt) continue
+      const version = await ctx.db.get(embedding.versionId)
+      const publicSoul = toPublicSoul(soul)
+      if (!publicSoul) continue
+      entries.push({ embeddingId, soul: publicSoul, version })
+    }
+
+    return entries
+  },
+})
+
+export const getSkillBadgeMapsInternal = internalQuery({
+  args: { skillIds: v.array(v.id('skills')) },
+  handler: async (ctx, args) => {
+    const badgeMap = await getSkillBadgeMaps(ctx, args.skillIds)
+    return Array.from(badgeMap.entries())
+  },
+})
+
+export const __test = { getNextCandidateLimit }

package/convex/seed.test.ts

@@ -0,0 +1,37 @@
+import { describe, expect, it } from 'vitest'
+import type { Doc } from './_generated/dataModel'
+import { decideSeedStart } from './seed'
+
+function seedState(cursor: string, updatedAt: number) {
+  return { cursor, updatedAt } as unknown as Doc<'githubBackupSyncState'>
+}
+
+describe('decideSeedStart', () => {
+  it('returns done when done', () => {
+    expect(decideSeedStart(seedState('done', Date.now()), Date.now())).toEqual({
+      started: false,
+      reason: 'done',
+    })
+  })
+
+  it('returns running when lock fresh', () => {
+    const now = Date.now()
+    expect(decideSeedStart(seedState('running', now), now + 1000)).toEqual({
+      started: false,
+      reason: 'running',
+    })
+  })
+
+  it('starts when lock stale', () => {
+    const now = Date.now()
+    const stale = now - 10 * 60 * 1000 - 1
+    expect(decideSeedStart(seedState('running', stale), now)).toEqual({
+      started: true,
+      reason: 'patched',
+    })
+  })
+
+  it('starts when missing', () => {
+    expect(decideSeedStart(null, Date.now())).toEqual({ started: true, reason: 'inserted' })
+  })
+})

package/convex/seed.ts

@@ -0,0 +1,254 @@
+import { v } from 'convex/values'
+import { internal } from './_generated/api'
+import type { Doc, Id } from './_generated/dataModel'
+import type { ActionCtx, DatabaseReader, DatabaseWriter } from './_generated/server'
+import { action, internalMutation, internalQuery } from './_generated/server'
+import { publishSoulVersionForUser } from './lib/soulPublish'
+import { SOUL_SEED_DISPLAY_NAME, SOUL_SEED_HANDLE, SOUL_SEED_KEY, SOUL_SEEDS } from './seedSouls'
+
+const SEED_LOCK_STALE_MS = 10 * 60 * 1000
+
+type SeedStateDoc = Doc<'githubBackupSyncState'>
+
+type SeedStartDecision = {
+  started: boolean
+  reason: 'done' | 'running' | 'patched' | 'inserted'
+}
+
+async function getSeedState(ctx: { db: DatabaseReader }): Promise<SeedStateDoc | null> {
+  const entries = (await ctx.db
+    .query('githubBackupSyncState')
+    .withIndex('by_key', (q) => q.eq('key', SOUL_SEED_KEY))
+    .order('desc')
+    .take(2)) as SeedStateDoc[]
+  return entries[0] ?? null
+}
+
+async function cleanupSeedState(ctx: { db: DatabaseWriter }, keepId: Id<'githubBackupSyncState'>) {
+  const entries = (await ctx.db
+    .query('githubBackupSyncState')
+    .withIndex('by_key', (q) => q.eq('key', SOUL_SEED_KEY))
+    .order('desc')
+    .take(50)) as SeedStateDoc[]
+
+  for (const entry of entries) {
+    if (entry._id === keepId) continue
+    await ctx.db.delete(entry._id)
+  }
+}
+
+export function decideSeedStart(existing: SeedStateDoc | null, now: number): SeedStartDecision {
+  const cursor = existing?.cursor ?? null
+  if (cursor === 'done') return { started: false, reason: 'done' }
+  if (cursor === 'running' && existing && now - existing.updatedAt < SEED_LOCK_STALE_MS) {
+    return { started: false, reason: 'running' }
+  }
+  return existing ? { started: true, reason: 'patched' } : { started: true, reason: 'inserted' }
+}
+
+export const getSoulSeedStateInternal = internalQuery({
+  args: {},
+  handler: async (ctx) => getSeedState(ctx),
+})
+
+export const setSoulSeedStateInternal = internalMutation({
+  args: { status: v.string() },
+  handler: async (ctx, args) => {
+    const existing = await getSeedState(ctx)
+    const now = Date.now()
+    if (existing) {
+      await ctx.db.patch(existing._id, { cursor: args.status, updatedAt: now })
+      await cleanupSeedState(ctx, existing._id)
+      return existing._id
+    }
+    const id = await ctx.db.insert('githubBackupSyncState', {
+      key: SOUL_SEED_KEY,
+      cursor: args.status,
+      updatedAt: now,
+    })
+    await cleanupSeedState(ctx, id)
+    return id
+  },
+})
+
+export const tryStartSoulSeedInternal = internalMutation({
+  args: {},
+  handler: async (ctx) => {
+    const now = Date.now()
+    const existing = await getSeedState(ctx)
+    const decision = decideSeedStart(existing, now)
+
+    if (!decision.started) return decision
+
+    if (existing) {
+      await ctx.db.patch(existing._id, { cursor: 'running', updatedAt: now })
+      await cleanupSeedState(ctx, existing._id)
+      return { started: true, reason: 'patched' as const }
+    }
+
+    const id = await ctx.db.insert('githubBackupSyncState', {
+      key: SOUL_SEED_KEY,
+      cursor: 'running',
+      updatedAt: now,
+    })
+    await cleanupSeedState(ctx, id)
+    return { started: true, reason: 'inserted' as const }
+  },
+})
+
+export const hasAnySoulsInternal = internalQuery({
+  args: {},
+  handler: async (ctx) => {
+    const entry = await ctx.db.query('souls').take(1)
+    return entry.length > 0
+  },
+})
+
+export const ensureSoulSeeds = action({
+  args: {},
+  handler: async (ctx) => {
+    const started = (await ctx.runMutation(internal.seed.tryStartSoulSeedInternal, {})) as {
+      started: boolean
+      reason: 'done' | 'running' | 'patched' | 'inserted'
+    }
+    if (!started.started) {
+      if (started.reason === 'done') return { seeded: false, reason: 'already-seeded' as const }
+      return { seeded: false, reason: 'in-progress' as const }
+    }
+
+    const hasSouls = (await ctx.runQuery(internal.seed.hasAnySoulsInternal, {})) as boolean
+    if (hasSouls) {
+      await ctx.runMutation(internal.seed.setSoulSeedStateInternal, { status: 'done' })
+      return { seeded: false, reason: 'souls-exist' as const }
+    }
+
+    try {
+      const result = await runSeed(ctx)
+      await ctx.runMutation(internal.seed.setSoulSeedStateInternal, { status: 'done' })
+      return { seeded: true, reason: 'seeded' as const, ...result }
+    } catch (error) {
+      await ctx.runMutation(internal.seed.setSoulSeedStateInternal, { status: 'error' })
+      throw error
+    }
+  },
+})
+
+export const seed = action({
+  args: {},
+  handler: async (ctx) => runSeed(ctx),
+})
+
+async function runSeed(ctx: ActionCtx) {
+  const userId = (await ctx.runMutation(internal.seed.ensureSeedUserInternal, {
+    handle: SOUL_SEED_HANDLE,
+    displayName: SOUL_SEED_DISPLAY_NAME,
+  })) as Id<'users'>
+
+  const created: string[] = []
+  const skipped: string[] = []
+
+  for (const seedEntry of SOUL_SEEDS) {
+    const existing = (await ctx.runQuery(internal.souls.getSoulBySlugInternal, {
+      slug: seedEntry.slug,
+    })) as Doc<'souls'> | null
+    if (existing) {
+      if (existing.softDeletedAt && existing.ownerUserId === userId) {
+        await ctx.runMutation(internal.souls.setSoulSoftDeletedInternal, {
+          userId,
+          slug: seedEntry.slug,
+          deleted: false,
+        })
+      }
+      skipped.push(seedEntry.slug)
+      continue
+    }
+
+    const body = seedEntry.readme
+    if (!body) {
+      skipped.push(seedEntry.slug)
+      continue
+    }
+
+    const bytes = new TextEncoder().encode(body)
+    const sha256 = await sha256Hex(bytes)
+    const storageId = await ctx.storage.store(new Blob([bytes], { type: 'text/markdown' }))
+
+    try {
+      await publishSoulVersionForUser(ctx, userId, {
+        slug: seedEntry.slug,
+        displayName: seedEntry.displayName,
+        version: seedEntry.version,
+        changelog: '',
+        tags: seedEntry.tags,
+        files: [
+          {
+            path: 'SOUL.md',
+            size: bytes.byteLength,
+            storageId,
+            sha256,
+            contentType: 'text/markdown',
+          },
+        ],
+      })
+      created.push(seedEntry.slug)
+    } catch (error) {
+      if (!isExpectedSeedSkipError(error)) throw error
+      skipped.push(seedEntry.slug)
+    }
+  }
+
+  return { created, skipped }
+}
+
+function isExpectedSeedSkipError(error: unknown) {
+  const message = error instanceof Error ? error.message : String(error)
+  return (
+    message.includes('Version already exists') || message.includes('Only the owner can publish')
+  )
+}
+
+export const ensureSeedUserInternal = internalMutation({
+  args: {
+    handle: v.string(),
+    displayName: v.string(),
+  },
+  handler: async (ctx, args) => {
+    const baseHandle = args.handle.trim()
+    const displayName = args.displayName.trim()
+    const candidates = [baseHandle, `${baseHandle}-bot`]
+    for (let i = 2; i <= 6; i += 1) candidates.push(`${baseHandle}-bot-${i}`)
+
+    for (const candidate of candidates) {
+      const existing = await ctx.db
+        .query('users')
+        .withIndex('handle', (q) => q.eq('handle', candidate))
+        .take(2)
+      const user = (existing[0] ?? null) as Doc<'users'> | null
+      if (user) {
+        if ((user.displayName ?? user.name) === displayName) return user._id
+        continue
+      }
+
+      return ctx.db.insert('users', {
+        handle: candidate,
+        displayName,
+        createdAt: Date.now(),
+        updatedAt: Date.now(),
+      })
+    }
+
+    throw new Error('Unable to allocate seed user handle')
+  },
+})
+
+async function sha256Hex(bytes: Uint8Array) {
+  const data = new Uint8Array(bytes)
+  const digest = await crypto.subtle.digest('SHA-256', data)
+  return toHex(new Uint8Array(digest))
+}
+
+function toHex(bytes: Uint8Array) {
+  let out = ''
+  for (const byte of bytes) out += byte.toString(16).padStart(2, '0')
+  return out
+}