pilothub 0.0.1 → 0.0.2

package/convex/search.ts

@@ -1,254 +0,0 @@
-import { v } from 'convex/values'
-import { internal } from './_generated/api'
-import type { Doc, Id } from './_generated/dataModel'
-import { action, internalQuery } from './_generated/server'
-import { getSkillBadgeMaps, isSkillHighlighted, type SkillBadgeMap } from './lib/badges'
-import { generateEmbedding } from './lib/embeddings'
-import { toPublicSkill, toPublicSoul } from './lib/public'
-import { matchesExactTokens, tokenize } from './lib/searchText'
-
-type HydratedEntry = {
-  embeddingId: Id<'skillEmbeddings'>
-  skill: NonNullable<ReturnType<typeof toPublicSkill>>
-  version: Doc<'skillVersions'> | null
-  ownerHandle: string | null
-}
-
-type SearchResult = HydratedEntry & { score: number }
-
-function getNextCandidateLimit(current: number, max: number) {
-  const next = Math.min(current * 2, max)
-  return next > current ? next : null
-}
-
-export const searchSkills: ReturnType<typeof action> = action({
-  args: {
-    query: v.string(),
-    limit: v.optional(v.number()),
-    highlightedOnly: v.optional(v.boolean()),
-  },
-  handler: async (ctx, args): Promise<SearchResult[]> => {
-    const query = args.query.trim()
-    if (!query) return []
-    const queryTokens = tokenize(query)
-    if (queryTokens.length === 0) return []
-    let vector: number[]
-    try {
-      vector = await generateEmbedding(query)
-    } catch (error) {
-      console.warn('Search embedding generation failed', error)
-      return []
-    }
-    const limit = args.limit ?? 10
-    // Convex vectorSearch max limit is 256; clamp candidate sizes accordingly.
-    const maxCandidate = Math.min(Math.max(limit * 10, 200), 256)
-    let candidateLimit = Math.min(Math.max(limit * 3, 50), 256)
-    let hydrated: HydratedEntry[] = []
-    let scoreById = new Map<Id<'skillEmbeddings'>, number>()
-    let exactMatches: HydratedEntry[] = []
-
-    while (candidateLimit <= maxCandidate) {
-      const results = await ctx.vectorSearch('skillEmbeddings', 'by_embedding', {
-        vector,
-        limit: candidateLimit,
-        filter: (q) => q.or(q.eq('visibility', 'latest'), q.eq('visibility', 'latest-approved')),
-      })
-
-      hydrated = (await ctx.runQuery(internal.search.hydrateResults, {
-        embeddingIds: results.map((result) => result._id),
-      })) as HydratedEntry[]
-
-      scoreById = new Map<Id<'skillEmbeddings'>, number>(
-        results.map((result) => [result._id, result._score]),
-      )
-
-      const badgeMapEntries = (await ctx.runQuery(internal.search.getSkillBadgeMapsInternal, {
-        skillIds: hydrated.map((entry) => entry.skill._id),
-      })) as Array<[Id<'skills'>, SkillBadgeMap]>
-      const badgeMapBySkillId = new Map(badgeMapEntries)
-      const hydratedWithBadges = hydrated.map((entry) => ({
-        ...entry,
-        skill: {
-          ...entry.skill,
-          badges: badgeMapBySkillId.get(entry.skill._id) ?? {},
-        },
-      }))
-
-      const filtered = args.highlightedOnly
-        ? hydratedWithBadges.filter((entry) => isSkillHighlighted(entry.skill))
-        : hydratedWithBadges
-
-      exactMatches = filtered.filter((entry) =>
-        matchesExactTokens(queryTokens, [
-          entry.skill.displayName,
-          entry.skill.slug,
-          entry.skill.summary,
-        ]),
-      )
-
-      if (exactMatches.length >= limit || results.length < candidateLimit) {
-        break
-      }
-
-      const nextLimit = getNextCandidateLimit(candidateLimit, maxCandidate)
-      if (!nextLimit) break
-      candidateLimit = nextLimit
-    }
-
-    return exactMatches
-      .map((entry) => ({
-        ...entry,
-        score: scoreById.get(entry.embeddingId) ?? 0,
-      }))
-      .filter((entry) => entry.skill)
-      .slice(0, limit)
-  },
-})
-
-export const getBadgeMapsForSkills = internalQuery({
-  args: { skillIds: v.array(v.id('skills')) },
-  handler: async (ctx, args): Promise<Array<[Id<'skills'>, SkillBadgeMap]>> => {
-    const badgeMap = await getSkillBadgeMaps(ctx, args.skillIds)
-    return Array.from(badgeMap.entries())
-  },
-})
-
-export const hydrateResults = internalQuery({
-  args: { embeddingIds: v.array(v.id('skillEmbeddings')) },
-  handler: async (ctx, args): Promise<HydratedEntry[]> => {
-    const ownerHandleCache = new Map<Id<'users'>, Promise<string | null>>()
-
-    const getOwnerHandle = (ownerUserId: Id<'users'>) => {
-      const cached = ownerHandleCache.get(ownerUserId)
-      if (cached) return cached
-      const handlePromise = ctx.db
-        .get(ownerUserId)
-        .then((owner) => owner?.handle ?? owner?._id ?? null)
-      ownerHandleCache.set(ownerUserId, handlePromise)
-      return handlePromise
-    }
-
-    const entries = await Promise.all(
-      args.embeddingIds.map(async (embeddingId) => {
-        const embedding = await ctx.db.get(embeddingId)
-        if (!embedding) return null
-        const skill = await ctx.db.get(embedding.skillId)
-        if (!skill || skill.softDeletedAt) return null
-        const [version, ownerHandle] = await Promise.all([
-          ctx.db.get(embedding.versionId),
-          getOwnerHandle(skill.ownerUserId),
-        ])
-        const publicSkill = toPublicSkill(skill)
-        if (!publicSkill) return null
-        return { embeddingId, skill: publicSkill, version, ownerHandle }
-      }),
-    )
-
-    return entries.filter((entry): entry is HydratedEntry => entry !== null)
-  },
-})
-
-type HydratedSoulEntry = {
-  embeddingId: Id<'soulEmbeddings'>
-  soul: NonNullable<ReturnType<typeof toPublicSoul>>
-  version: Doc<'soulVersions'> | null
-}
-
-type SoulSearchResult = HydratedSoulEntry & { score: number }
-
-export const searchSouls: ReturnType<typeof action> = action({
-  args: {
-    query: v.string(),
-    limit: v.optional(v.number()),
-  },
-  handler: async (ctx, args): Promise<SoulSearchResult[]> => {
-    const query = args.query.trim()
-    if (!query) return []
-    const queryTokens = tokenize(query)
-    if (queryTokens.length === 0) return []
-    let vector: number[]
-    try {
-      vector = await generateEmbedding(query)
-    } catch (error) {
-      console.warn('Search embedding generation failed', error)
-      return []
-    }
-    const limit = args.limit ?? 10
-    // Convex vectorSearch max limit is 256; clamp candidate sizes accordingly.
-    const maxCandidate = Math.min(Math.max(limit * 10, 200), 256)
-    let candidateLimit = Math.min(Math.max(limit * 3, 50), 256)
-    let hydrated: HydratedSoulEntry[] = []
-    let scoreById = new Map<Id<'soulEmbeddings'>, number>()
-    let exactMatches: HydratedSoulEntry[] = []
-
-    while (candidateLimit <= maxCandidate) {
-      const results = await ctx.vectorSearch('soulEmbeddings', 'by_embedding', {
-        vector,
-        limit: candidateLimit,
-        filter: (q) => q.or(q.eq('visibility', 'latest'), q.eq('visibility', 'latest-approved')),
-      })
-
-      hydrated = (await ctx.runQuery(internal.search.hydrateSoulResults, {
-        embeddingIds: results.map((result) => result._id),
-      })) as HydratedSoulEntry[]
-
-      scoreById = new Map<Id<'soulEmbeddings'>, number>(
-        results.map((result) => [result._id, result._score]),
-      )
-
-      exactMatches = hydrated.filter((entry) =>
-        matchesExactTokens(queryTokens, [
-          entry.soul.displayName,
-          entry.soul.slug,
-          entry.soul.summary,
-        ]),
-      )
-
-      if (exactMatches.length >= limit || results.length < candidateLimit) {
-        break
-      }
-
-      const nextLimit = getNextCandidateLimit(candidateLimit, maxCandidate)
-      if (!nextLimit) break
-      candidateLimit = nextLimit
-    }
-
-    return exactMatches
-      .map((entry) => ({
-        ...entry,
-        score: scoreById.get(entry.embeddingId) ?? 0,
-      }))
-      .filter((entry) => entry.soul)
-      .slice(0, limit)
-  },
-})
-
-export const hydrateSoulResults = internalQuery({
-  args: { embeddingIds: v.array(v.id('soulEmbeddings')) },
-  handler: async (ctx, args): Promise<HydratedSoulEntry[]> => {
-    const entries: HydratedSoulEntry[] = []
-
-    for (const embeddingId of args.embeddingIds) {
-      const embedding = await ctx.db.get(embeddingId)
-      if (!embedding) continue
-      const soul = await ctx.db.get(embedding.soulId)
-      if (soul?.softDeletedAt) continue
-      const version = await ctx.db.get(embedding.versionId)
-      const publicSoul = toPublicSoul(soul)
-      if (!publicSoul) continue
-      entries.push({ embeddingId, soul: publicSoul, version })
-    }
-
-    return entries
-  },
-})
-
-export const getSkillBadgeMapsInternal = internalQuery({
-  args: { skillIds: v.array(v.id('skills')) },
-  handler: async (ctx, args) => {
-    const badgeMap = await getSkillBadgeMaps(ctx, args.skillIds)
-    return Array.from(badgeMap.entries())
-  },
-})
-
-export const __test = { getNextCandidateLimit }

package/convex/seed.test.ts

@@ -1,37 +0,0 @@
-import { describe, expect, it } from 'vitest'
-import type { Doc } from './_generated/dataModel'
-import { decideSeedStart } from './seed'
-
-function seedState(cursor: string, updatedAt: number) {
-  return { cursor, updatedAt } as unknown as Doc<'githubBackupSyncState'>
-}
-
-describe('decideSeedStart', () => {
-  it('returns done when done', () => {
-    expect(decideSeedStart(seedState('done', Date.now()), Date.now())).toEqual({
-      started: false,
-      reason: 'done',
-    })
-  })
-
-  it('returns running when lock fresh', () => {
-    const now = Date.now()
-    expect(decideSeedStart(seedState('running', now), now + 1000)).toEqual({
-      started: false,
-      reason: 'running',
-    })
-  })
-
-  it('starts when lock stale', () => {
-    const now = Date.now()
-    const stale = now - 10 * 60 * 1000 - 1
-    expect(decideSeedStart(seedState('running', stale), now)).toEqual({
-      started: true,
-      reason: 'patched',
-    })
-  })
-
-  it('starts when missing', () => {
-    expect(decideSeedStart(null, Date.now())).toEqual({ started: true, reason: 'inserted' })
-  })
-})

package/convex/seed.ts

@@ -1,254 +0,0 @@
-import { v } from 'convex/values'
-import { internal } from './_generated/api'
-import type { Doc, Id } from './_generated/dataModel'
-import type { ActionCtx, DatabaseReader, DatabaseWriter } from './_generated/server'
-import { action, internalMutation, internalQuery } from './_generated/server'
-import { publishSoulVersionForUser } from './lib/soulPublish'
-import { SOUL_SEED_DISPLAY_NAME, SOUL_SEED_HANDLE, SOUL_SEED_KEY, SOUL_SEEDS } from './seedSouls'
-
-const SEED_LOCK_STALE_MS = 10 * 60 * 1000
-
-type SeedStateDoc = Doc<'githubBackupSyncState'>
-
-type SeedStartDecision = {
-  started: boolean
-  reason: 'done' | 'running' | 'patched' | 'inserted'
-}
-
-async function getSeedState(ctx: { db: DatabaseReader }): Promise<SeedStateDoc | null> {
-  const entries = (await ctx.db
-    .query('githubBackupSyncState')
-    .withIndex('by_key', (q) => q.eq('key', SOUL_SEED_KEY))
-    .order('desc')
-    .take(2)) as SeedStateDoc[]
-  return entries[0] ?? null
-}
-
-async function cleanupSeedState(ctx: { db: DatabaseWriter }, keepId: Id<'githubBackupSyncState'>) {
-  const entries = (await ctx.db
-    .query('githubBackupSyncState')
-    .withIndex('by_key', (q) => q.eq('key', SOUL_SEED_KEY))
-    .order('desc')
-    .take(50)) as SeedStateDoc[]
-
-  for (const entry of entries) {
-    if (entry._id === keepId) continue
-    await ctx.db.delete(entry._id)
-  }
-}
-
-export function decideSeedStart(existing: SeedStateDoc | null, now: number): SeedStartDecision {
-  const cursor = existing?.cursor ?? null
-  if (cursor === 'done') return { started: false, reason: 'done' }
-  if (cursor === 'running' && existing && now - existing.updatedAt < SEED_LOCK_STALE_MS) {
-    return { started: false, reason: 'running' }
-  }
-  return existing ? { started: true, reason: 'patched' } : { started: true, reason: 'inserted' }
-}
-
-export const getSoulSeedStateInternal = internalQuery({
-  args: {},
-  handler: async (ctx) => getSeedState(ctx),
-})
-
-export const setSoulSeedStateInternal = internalMutation({
-  args: { status: v.string() },
-  handler: async (ctx, args) => {
-    const existing = await getSeedState(ctx)
-    const now = Date.now()
-    if (existing) {
-      await ctx.db.patch(existing._id, { cursor: args.status, updatedAt: now })
-      await cleanupSeedState(ctx, existing._id)
-      return existing._id
-    }
-    const id = await ctx.db.insert('githubBackupSyncState', {
-      key: SOUL_SEED_KEY,
-      cursor: args.status,
-      updatedAt: now,
-    })
-    await cleanupSeedState(ctx, id)
-    return id
-  },
-})
-
-export const tryStartSoulSeedInternal = internalMutation({
-  args: {},
-  handler: async (ctx) => {
-    const now = Date.now()
-    const existing = await getSeedState(ctx)
-    const decision = decideSeedStart(existing, now)
-
-    if (!decision.started) return decision
-
-    if (existing) {
-      await ctx.db.patch(existing._id, { cursor: 'running', updatedAt: now })
-      await cleanupSeedState(ctx, existing._id)
-      return { started: true, reason: 'patched' as const }
-    }
-
-    const id = await ctx.db.insert('githubBackupSyncState', {
-      key: SOUL_SEED_KEY,
-      cursor: 'running',
-      updatedAt: now,
-    })
-    await cleanupSeedState(ctx, id)
-    return { started: true, reason: 'inserted' as const }
-  },
-})
-
-export const hasAnySoulsInternal = internalQuery({
-  args: {},
-  handler: async (ctx) => {
-    const entry = await ctx.db.query('souls').take(1)
-    return entry.length > 0
-  },
-})
-
-export const ensureSoulSeeds = action({
-  args: {},
-  handler: async (ctx) => {
-    const started = (await ctx.runMutation(internal.seed.tryStartSoulSeedInternal, {})) as {
-      started: boolean
-      reason: 'done' | 'running' | 'patched' | 'inserted'
-    }
-    if (!started.started) {
-      if (started.reason === 'done') return { seeded: false, reason: 'already-seeded' as const }
-      return { seeded: false, reason: 'in-progress' as const }
-    }
-
-    const hasSouls = (await ctx.runQuery(internal.seed.hasAnySoulsInternal, {})) as boolean
-    if (hasSouls) {
-      await ctx.runMutation(internal.seed.setSoulSeedStateInternal, { status: 'done' })
-      return { seeded: false, reason: 'souls-exist' as const }
-    }
-
-    try {
-      const result = await runSeed(ctx)
-      await ctx.runMutation(internal.seed.setSoulSeedStateInternal, { status: 'done' })
-      return { seeded: true, reason: 'seeded' as const, ...result }
-    } catch (error) {
-      await ctx.runMutation(internal.seed.setSoulSeedStateInternal, { status: 'error' })
-      throw error
-    }
-  },
-})
-
-export const seed = action({
-  args: {},
-  handler: async (ctx) => runSeed(ctx),
-})
-
-async function runSeed(ctx: ActionCtx) {
-  const userId = (await ctx.runMutation(internal.seed.ensureSeedUserInternal, {
-    handle: SOUL_SEED_HANDLE,
-    displayName: SOUL_SEED_DISPLAY_NAME,
-  })) as Id<'users'>
-
-  const created: string[] = []
-  const skipped: string[] = []
-
-  for (const seedEntry of SOUL_SEEDS) {
-    const existing = (await ctx.runQuery(internal.souls.getSoulBySlugInternal, {
-      slug: seedEntry.slug,
-    })) as Doc<'souls'> | null
-    if (existing) {
-      if (existing.softDeletedAt && existing.ownerUserId === userId) {
-        await ctx.runMutation(internal.souls.setSoulSoftDeletedInternal, {
-          userId,
-          slug: seedEntry.slug,
-          deleted: false,
-        })
-      }
-      skipped.push(seedEntry.slug)
-      continue
-    }
-
-    const body = seedEntry.readme
-    if (!body) {
-      skipped.push(seedEntry.slug)
-      continue
-    }
-
-    const bytes = new TextEncoder().encode(body)
-    const sha256 = await sha256Hex(bytes)
-    const storageId = await ctx.storage.store(new Blob([bytes], { type: 'text/markdown' }))
-
-    try {
-      await publishSoulVersionForUser(ctx, userId, {
-        slug: seedEntry.slug,
-        displayName: seedEntry.displayName,
-        version: seedEntry.version,
-        changelog: '',
-        tags: seedEntry.tags,
-        files: [
-          {
-            path: 'SOUL.md',
-            size: bytes.byteLength,
-            storageId,
-            sha256,
-            contentType: 'text/markdown',
-          },
-        ],
-      })
-      created.push(seedEntry.slug)
-    } catch (error) {
-      if (!isExpectedSeedSkipError(error)) throw error
-      skipped.push(seedEntry.slug)
-    }
-  }
-
-  return { created, skipped }
-}
-
-function isExpectedSeedSkipError(error: unknown) {
-  const message = error instanceof Error ? error.message : String(error)
-  return (
-    message.includes('Version already exists') || message.includes('Only the owner can publish')
-  )
-}
-
-export const ensureSeedUserInternal = internalMutation({
-  args: {
-    handle: v.string(),
-    displayName: v.string(),
-  },
-  handler: async (ctx, args) => {
-    const baseHandle = args.handle.trim()
-    const displayName = args.displayName.trim()
-    const candidates = [baseHandle, `${baseHandle}-bot`]
-    for (let i = 2; i <= 6; i += 1) candidates.push(`${baseHandle}-bot-${i}`)
-
-    for (const candidate of candidates) {
-      const existing = await ctx.db
-        .query('users')
-        .withIndex('handle', (q) => q.eq('handle', candidate))
-        .take(2)
-      const user = (existing[0] ?? null) as Doc<'users'> | null
-      if (user) {
-        if ((user.displayName ?? user.name) === displayName) return user._id
-        continue
-      }
-
-      return ctx.db.insert('users', {
-        handle: candidate,
-        displayName,
-        createdAt: Date.now(),
-        updatedAt: Date.now(),
-      })
-    }
-
-    throw new Error('Unable to allocate seed user handle')
-  },
-})
-
-async function sha256Hex(bytes: Uint8Array) {
-  const data = new Uint8Array(bytes)
-  const digest = await crypto.subtle.digest('SHA-256', data)
-  return toHex(new Uint8Array(digest))
-}
-
-function toHex(bytes: Uint8Array) {
-  let out = ''
-  for (const byte of bytes) out += byte.toString(16).padStart(2, '0')
-  return out
-}