pilothub 0.0.1

package/convex/lib/searchText.test.ts

@@ -0,0 +1,46 @@
+/* @vitest-environment node */
+
+import { describe, expect, it } from 'vitest'
+import { __test, matchesExactTokens, tokenize } from './searchText'
+
+describe('searchText', () => {
+  it('tokenize lowercases and splits on punctuation', () => {
+    expect(tokenize('Minimax Usage /minimax-usage')).toEqual([
+      'minimax',
+      'usage',
+      'minimax',
+      'usage',
+    ])
+  })
+
+  it('matchesExactTokens requires at least one query token to prefix-match', () => {
+    const queryTokens = tokenize('Remind Me')
+    expect(matchesExactTokens(queryTokens, ['Remind Me', '/remind-me', 'Short summary'])).toBe(true)
+    // "Reminder" starts with "remind", so it matches with prefix matching
+    expect(matchesExactTokens(queryTokens, ['Reminder tool', '/reminder', 'Short summary'])).toBe(
+      true,
+    )
+    // Matches because "remind" token is present
+    expect(matchesExactTokens(queryTokens, ['Remind tool', '/remind', 'Short summary'])).toBe(true)
+    // No matching tokens at all
+    expect(matchesExactTokens(queryTokens, ['Other tool', '/other', 'Short summary'])).toBe(false)
+  })
+
+  it('matchesExactTokens supports prefix matching for partial queries', () => {
+    // "go" should match "gohome" because "gohome" starts with "go"
+    expect(matchesExactTokens(['go'], ['GoHome', '/gohome', 'Navigate home'])).toBe(true)
+    // "pad" should match "padel"
+    expect(matchesExactTokens(['pad'], ['Padel', '/padel', 'Tennis-like sport'])).toBe(true)
+    // "xyz" should not match anything
+    expect(matchesExactTokens(['xyz'], ['GoHome', '/gohome', 'Navigate home'])).toBe(false)
+  })
+
+  it('matchesExactTokens ignores empty inputs', () => {
+    expect(matchesExactTokens([], ['text'])).toBe(false)
+    expect(matchesExactTokens(['token'], ['  ', null, undefined])).toBe(false)
+  })
+
+  it('normalize uses lowercase', () => {
+    expect(__test.normalize('AbC')).toBe('abc')
+  })
+})

package/convex/lib/searchText.ts

@@ -0,0 +1,27 @@
+const WORD_RE = /[a-z0-9]+/g
+
+function normalize(value: string) {
+  return value.toLowerCase()
+}
+
+export function tokenize(value: string): string[] {
+  if (!value) return []
+  return normalize(value).match(WORD_RE) ?? []
+}
+
+export function matchesExactTokens(
+  queryTokens: string[],
+  parts: Array<string | null | undefined>,
+): boolean {
+  if (queryTokens.length === 0) return false
+  const text = parts.filter((part) => Boolean(part?.trim())).join(' ')
+  if (!text) return false
+  const textTokens = tokenize(text)
+  if (textTokens.length === 0) return false
+  // Require at least one token to prefix-match, allowing vector similarity to determine relevance
+  return queryTokens.some((queryToken) =>
+    textTokens.some((textToken) => textToken.includes(queryToken)),
+  )
+}
+
+export const __test = { normalize, tokenize, matchesExactTokens }

package/convex/lib/skillBackfill.test.ts

@@ -0,0 +1,34 @@
+import { describe, expect, it } from 'vitest'
+import { buildSkillSummaryBackfillPatch } from './skillBackfill'
+
+describe('skill backfill', () => {
+  it('produces summary + parsed patch from block scalar', () => {
+    const patch = buildSkillSummaryBackfillPatch({
+      readmeText: `---\ndescription: >\n  Hello\n  world.\n---\nBody`,
+      currentSummary: '>',
+      currentParsed: { frontmatter: { description: '>' } },
+    })
+    expect(patch.summary).toBe('Hello world.')
+    expect(patch.parsed?.frontmatter.description).toBe('Hello world.')
+  })
+
+  it('does not set summary when description is not a string', () => {
+    const patch = buildSkillSummaryBackfillPatch({
+      readmeText: `---\ndescription:\n  - a\n---\nBody`,
+      currentSummary: 'Old',
+      currentParsed: { frontmatter: {} },
+    })
+    expect(patch.summary).toBeUndefined()
+    expect(patch.parsed?.frontmatter.description).toEqual(['a'])
+  })
+
+  it('keeps legacy summary when unchanged and still updates parsed', () => {
+    const patch = buildSkillSummaryBackfillPatch({
+      readmeText: `---\ndescription: Hello\n---\nBody`,
+      currentSummary: 'Hello',
+      currentParsed: { frontmatter: { description: 'nope' } },
+    })
+    expect(patch.summary).toBeUndefined()
+    expect(patch.parsed?.frontmatter.description).toBe('Hello')
+  })
+})

package/convex/lib/skillBackfill.ts

@@ -0,0 +1,67 @@
+import {
+  getFrontmatterMetadata,
+  getFrontmatterValue,
+  type ParsedSkillFrontmatter,
+  parseFrontmatter,
+  parsePilotbotMetadata,
+} from './skills'
+
+export type ParsedSkillData = {
+  frontmatter: ParsedSkillFrontmatter
+  metadata?: unknown
+  pilotbot?: unknown
+}
+
+export type SkillSummaryBackfillPatch = {
+  summary?: string
+  parsed?: ParsedSkillData
+}
+
+export function buildSkillSummaryBackfillPatch(args: {
+  readmeText: string
+  currentSummary?: string
+  currentParsed?: ParsedSkillData
+}): SkillSummaryBackfillPatch {
+  const frontmatter = parseFrontmatter(args.readmeText)
+  const summary = getFrontmatterValue(frontmatter, 'description') ?? undefined
+  const metadata = getFrontmatterMetadata(frontmatter)
+  const pilotbot = parsePilotbotMetadata(frontmatter)
+  const parsed: ParsedSkillData = { frontmatter, metadata, pilotbot }
+
+  const patch: SkillSummaryBackfillPatch = {}
+  if (summary && summary !== args.currentSummary) {
+    patch.summary = summary
+  }
+  if (!deepEqual(parsed, args.currentParsed)) {
+    patch.parsed = parsed
+  }
+  return patch
+}
+
+function deepEqual(a: unknown, b: unknown): boolean {
+  if (a === b) return true
+  if (!a || !b) return a === b
+  if (typeof a !== typeof b) return false
+  if (Array.isArray(a) || Array.isArray(b)) {
+    if (!Array.isArray(a) || !Array.isArray(b)) return false
+    if (a.length !== b.length) return false
+    for (let i = 0; i < a.length; i++) {
+      if (!deepEqual(a[i], b[i])) return false
+    }
+    return true
+  }
+  if (typeof a === 'object' && typeof b === 'object') {
+    const aObj = a as Record<string, unknown>
+    const bObj = b as Record<string, unknown>
+    const aKeys = Object.keys(aObj).sort()
+    const bKeys = Object.keys(bObj).sort()
+    if (aKeys.length !== bKeys.length) return false
+    for (let i = 0; i < aKeys.length; i++) {
+      if (aKeys[i] !== bKeys[i]) return false
+      const key = aKeys[i] as string
+      if (!deepEqual(aObj[key], bObj[key])) return false
+    }
+    return true
+  }
+  return false
+}

package/convex/lib/skillPublish.test.ts

@@ -0,0 +1,28 @@
+import { describe, expect, it } from 'vitest'
+import { __test } from './skillPublish'
+
+describe('skillPublish', () => {
+  it('merges github source into metadata', () => {
+    const merged = __test.mergeSourceIntoMetadata(
+      { pilotbot: { emoji: 'x' } },
+      {
+        kind: 'github',
+        url: 'https://github.com/a/b',
+        repo: 'a/b',
+        ref: 'main',
+        commit: '0123456789012345678901234567890123456789',
+        path: 'skills/demo',
+        importedAt: 123,
+      },
+    )
+    expect((merged as Record<string, unknown>).pilotbot).toEqual({ emoji: 'x' })
+    const source = (merged as Record<string, unknown>).source
+    expect(source).toEqual(
+      expect.objectContaining({
+        kind: 'github',
+        repo: 'a/b',
+        path: 'skills/demo',
+      }),
+    )
+  })
+})

package/convex/lib/skillPublish.ts

@@ -0,0 +1,284 @@
+import { ConvexError } from 'convex/values'
+import semver from 'semver'
+import { api, internal } from '../_generated/api'
+import type { Doc, Id } from '../_generated/dataModel'
+import type { ActionCtx, MutationCtx } from '../_generated/server'
+import { getSkillBadgeMap, isSkillHighlighted } from './badges'
+import { generateChangelogForPublish } from './changelog'
+import { generateEmbedding } from './embeddings'
+import type { PublicUser } from './public'
+import {
+  buildEmbeddingText,
+  getFrontmatterMetadata,
+  hashSkillFiles,
+  isTextFile,
+  parseFrontmatter,
+  parsePilotbotMetadata,
+  sanitizePath,
+} from './skills'
+import type { WebhookSkillPayload } from './webhooks'
+
+const MAX_TOTAL_BYTES = 50 * 1024 * 1024
+const MAX_FILES_FOR_EMBEDDING = 40
+
+export type PublishResult = {
+  skillId: Id<'skills'>
+  versionId: Id<'skillVersions'>
+  embeddingId: Id<'skillEmbeddings'>
+}
+
+export type PublishVersionArgs = {
+  slug: string
+  displayName: string
+  version: string
+  changelog: string
+  tags?: string[]
+  forkOf?: { slug: string; version?: string }
+  source?: {
+    kind: 'github'
+    url: string
+    repo: string
+    ref: string
+    commit: string
+    path: string
+    importedAt: number
+  }
+  files: Array<{
+    path: string
+    size: number
+    storageId: Id<'_storage'>
+    sha256: string
+    contentType?: string
+  }>
+}
+
+export async function publishVersionForUser(
+  ctx: ActionCtx,
+  userId: Id<'users'>,
+  args: PublishVersionArgs,
+): Promise<PublishResult> {
+  const version = args.version.trim()
+  const slug = args.slug.trim().toLowerCase()
+  const displayName = args.displayName.trim()
+  if (!slug || !displayName) throw new ConvexError('Slug and display name required')
+  if (!/^[a-z0-9][a-z0-9-]*$/.test(slug)) {
+    throw new ConvexError('Slug must be lowercase and url-safe')
+  }
+  if (!semver.valid(version)) {
+    throw new ConvexError('Version must be valid semver')
+  }
+  const suppliedChangelog = args.changelog.trim()
+  const changelogSource = suppliedChangelog ? ('user' as const) : ('auto' as const)
+
+  const sanitizedFiles = args.files.map((file) => ({
+    ...file,
+    path: sanitizePath(file.path),
+  }))
+  if (sanitizedFiles.some((file) => !file.path)) {
+    throw new ConvexError('Invalid file paths')
+  }
+  const safeFiles = sanitizedFiles.map((file) => ({
+    ...file,
+    path: file.path as string,
+  }))
+  if (safeFiles.some((file) => !isTextFile(file.path, file.contentType ?? undefined))) {
+    throw new ConvexError('Only text-based files are allowed')
+  }
+
+  const totalBytes = safeFiles.reduce((sum, file) => sum + file.size, 0)
+  if (totalBytes > MAX_TOTAL_BYTES) {
+    throw new ConvexError('Skill bundle exceeds 50MB limit')
+  }
+
+  const readmeFile = safeFiles.find(
+    (file) => file.path?.toLowerCase() === 'skill.md' || file.path?.toLowerCase() === 'skills.md',
+  )
+  if (!readmeFile) throw new ConvexError('SKILL.md is required')
+
+  const readmeText = await fetchText(ctx, readmeFile.storageId)
+  const frontmatter = parseFrontmatter(readmeText)
+  const pilotbot = parsePilotbotMetadata(frontmatter)
+  const metadata = mergeSourceIntoMetadata(getFrontmatterMetadata(frontmatter), args.source)
+
+  const otherFiles = [] as Array<{ path: string; content: string }>
+  for (const file of safeFiles) {
+    if (!file.path || file.path.toLowerCase().endsWith('.md')) continue
+    if (!isTextFile(file.path, file.contentType ?? undefined)) continue
+    const content = await fetchText(ctx, file.storageId)
+    otherFiles.push({ path: file.path, content })
+    if (otherFiles.length >= MAX_FILES_FOR_EMBEDDING) break
+  }
+
+  const embeddingText = buildEmbeddingText({
+    frontmatter,
+    readme: readmeText,
+    otherFiles,
+  })
+
+  const fingerprintPromise = hashSkillFiles(
+    safeFiles.map((file) => ({ path: file.path, sha256: file.sha256 })),
+  )
+
+  const changelogPromise =
+    changelogSource === 'user'
+      ? Promise.resolve(suppliedChangelog)
+      : generateChangelogForPublish(ctx, {
+          slug,
+          version,
+          readmeText,
+          files: safeFiles.map((file) => ({ path: file.path, sha256: file.sha256 })),
+        })
+
+  const embeddingPromise = generateEmbedding(embeddingText)
+
+  const [fingerprint, changelogText, embedding] = await Promise.all([
+    fingerprintPromise,
+    changelogPromise,
+    embeddingPromise.catch((error) => {
+      throw new ConvexError(formatEmbeddingError(error))
+    }),
+  ])
+
+  const publishResult = (await ctx.runMutation(internal.skills.insertVersion, {
+    userId,
+    slug,
+    displayName,
+    version,
+    changelog: changelogText,
+    changelogSource,
+    tags: args.tags?.map((tag) => tag.trim()).filter(Boolean),
+    fingerprint,
+    forkOf: args.forkOf
+      ? {
+          slug: args.forkOf.slug.trim().toLowerCase(),
+          version: args.forkOf.version?.trim() || undefined,
+        }
+      : undefined,
+    files: safeFiles.map((file) => ({
+      ...file,
+      path: file.path,
+    })),
+    parsed: {
+      frontmatter,
+      metadata,
+      pilotbot,
+    },
+    embedding,
+  })) as PublishResult
+
+  const owner = (await ctx.runQuery(internal.users.getByIdInternal, {
+    userId,
+  })) as Doc<'users'> | null
+  const ownerHandle = owner?.handle ?? owner?.displayName ?? owner?.name ?? 'unknown'
+
+  void ctx.scheduler
+    .runAfter(0, internal.githubBackupsNode.backupSkillForPublishInternal, {
+      slug,
+      version,
+      displayName,
+      ownerHandle,
+      files: safeFiles,
+      publishedAt: Date.now(),
+    })
+    .catch((error) => {
+      console.error('GitHub backup scheduling failed', error)
+    })
+
+  void schedulePublishWebhook(ctx, {
+    slug,
+    version,
+    displayName,
+  })
+
+  return publishResult
+}
+
+function mergeSourceIntoMetadata(metadata: unknown, source: PublishVersionArgs['source']) {
+  if (!source) return metadata === undefined ? undefined : metadata
+  const sourceValue = {
+    kind: source.kind,
+    url: source.url,
+    repo: source.repo,
+    ref: source.ref,
+    commit: source.commit,
+    path: source.path,
+    importedAt: source.importedAt,
+  }
+
+  if (!metadata) return { source: sourceValue }
+  if (typeof metadata !== 'object' || Array.isArray(metadata)) return { source: sourceValue }
+  return { ...(metadata as Record<string, unknown>), source: sourceValue }
+}
+
+export const __test = {
+  mergeSourceIntoMetadata,
+}
+
+export async function queueHighlightedWebhook(ctx: MutationCtx, skillId: Id<'skills'>) {
+  const skill = await ctx.db.get(skillId)
+  if (!skill) return
+  const owner = await ctx.db.get(skill.ownerUserId)
+  const latestVersion = skill.latestVersionId ? await ctx.db.get(skill.latestVersionId) : null
+
+  const badges = await getSkillBadgeMap(ctx, skillId)
+  const payload: WebhookSkillPayload = {
+    slug: skill.slug,
+    displayName: skill.displayName,
+    summary: skill.summary ?? undefined,
+    version: latestVersion?.version ?? undefined,
+    ownerHandle: owner?.handle ?? owner?.name ?? undefined,
+    highlighted: isSkillHighlighted({ badges }),
+    tags: Object.keys(skill.tags ?? {}),
+  }
+
+  await ctx.scheduler.runAfter(0, internal.webhooks.sendDiscordWebhook, {
+    event: 'skill.highlighted',
+    skill: payload,
+  })
+}
+
+export async function fetchText(
+  ctx: { storage: { get: (id: Id<'_storage'>) => Promise<Blob | null> } },
+  storageId: Id<'_storage'>,
+) {
+  const blob = await ctx.storage.get(storageId)
+  if (!blob) throw new Error('File missing in storage')
+  return blob.text()
+}
+
+function formatEmbeddingError(error: unknown) {
+  if (error instanceof Error) {
+    if (error.message.includes('OPENAI_API_KEY')) {
+      return 'OPENAI_API_KEY is not configured.'
+    }
+    if (error.message.startsWith('Embedding failed')) {
+      return error.message
+    }
+  }
+  return 'Embedding failed. Please try again.'
+}
+
+async function schedulePublishWebhook(
+  ctx: ActionCtx,
+  params: { slug: string; version: string; displayName: string },
+) {
+  const result = (await ctx.runQuery(api.skills.getBySlug, {
+    slug: params.slug,
+  })) as { skill: Doc<'skills'>; owner: PublicUser | null } | null
+  if (!result?.skill) return
+
+  const payload: WebhookSkillPayload = {
+    slug: result.skill.slug,
+    displayName: result.skill.displayName || params.displayName,
+    summary: result.skill.summary ?? undefined,
+    version: params.version,
+    ownerHandle: result.owner?.handle ?? result.owner?.name ?? undefined,
+    highlighted: isSkillHighlighted(result.skill),
+    tags: Object.keys(result.skill.tags ?? {}),
+  }
+
+  await ctx.scheduler.runAfter(0, internal.webhooks.sendDiscordWebhook, {
+    event: 'skill.publish',
+    skill: payload,
+  })
+}

package/convex/lib/skillStats.ts

@@ -0,0 +1,80 @@
+import type { Doc, Id } from '../_generated/dataModel'
+import type { MutationCtx } from '../_generated/server'
+import { toDayKey } from './leaderboards'
+
+type SkillStatDeltas = {
+  downloads?: number
+  stars?: number
+  installsCurrent?: number
+  installsAllTime?: number
+}
+
+export function applySkillStatDeltas(skill: Doc<'skills'>, deltas: SkillStatDeltas) {
+  const currentDownloads =
+    typeof skill.statsDownloads === 'number' ? skill.statsDownloads : skill.stats.downloads
+  const currentStars = typeof skill.statsStars === 'number' ? skill.statsStars : skill.stats.stars
+  const currentInstallsCurrent =
+    typeof skill.statsInstallsCurrent === 'number'
+      ? skill.statsInstallsCurrent
+      : (skill.stats.installsCurrent ?? 0)
+  const currentInstallsAllTime =
+    typeof skill.statsInstallsAllTime === 'number'
+      ? skill.statsInstallsAllTime
+      : (skill.stats.installsAllTime ?? 0)
+
+  const nextDownloads = Math.max(0, currentDownloads + (deltas.downloads ?? 0))
+  const nextStars = Math.max(0, currentStars + (deltas.stars ?? 0))
+  const nextInstallsCurrent = Math.max(0, currentInstallsCurrent + (deltas.installsCurrent ?? 0))
+  const nextInstallsAllTime = Math.max(0, currentInstallsAllTime + (deltas.installsAllTime ?? 0))
+
+  return {
+    statsDownloads: nextDownloads,
+    statsStars: nextStars,
+    statsInstallsCurrent: nextInstallsCurrent,
+    statsInstallsAllTime: nextInstallsAllTime,
+    stats: {
+      ...skill.stats,
+      downloads: nextDownloads,
+      stars: nextStars,
+      installsCurrent: nextInstallsCurrent,
+      installsAllTime: nextInstallsAllTime,
+    },
+  }
+}
+
+export async function bumpDailySkillStats(
+  ctx: MutationCtx,
+  params: {
+    skillId: Id<'skills'>
+    now: number
+    downloads?: number
+    installs?: number
+  },
+) {
+  const downloads = params.downloads ?? 0
+  const installs = params.installs ?? 0
+  if (downloads === 0 && installs === 0) return
+
+  const day = toDayKey(params.now)
+  const existing = await ctx.db
+    .query('skillDailyStats')
+    .withIndex('by_skill_day', (q) => q.eq('skillId', params.skillId).eq('day', day))
+    .unique()
+
+  if (existing) {
+    await ctx.db.patch(existing._id, {
+      downloads: Math.max(0, existing.downloads + downloads),
+      installs: Math.max(0, existing.installs + installs),
+      updatedAt: params.now,
+    })
+    return
+  }
+
+  await ctx.db.insert('skillDailyStats', {
+    skillId: params.skillId,
+    day,
+    downloads: Math.max(0, downloads),
+    installs: Math.max(0, installs),
+    updatedAt: params.now,
+  })
+}