pilothub 0.0.1

package/convex/lib/soulChangelog.ts

@@ -0,0 +1,273 @@
+import { internal } from '../_generated/api'
+import type { Doc } from '../_generated/dataModel'
+import type { ActionCtx } from '../_generated/server'
+
+const CHANGELOG_MODEL = process.env.OPENAI_CHANGELOG_MODEL ?? 'gpt-4.1'
+const MAX_README_CHARS = 8_000
+const MAX_PATHS_IN_PROMPT = 30
+
+type FileMeta = { path: string; sha256?: string }
+
+type FileDiffSummary = {
+  added: string[]
+  removed: string[]
+  changed: string[]
+}
+
+function clampText(value: string, maxChars: number) {
+  const trimmed = value.trim()
+  if (trimmed.length <= maxChars) return trimmed
+  return `${trimmed.slice(0, maxChars).trimEnd()}\n…`
+}
+
+function summarizeFileDiff(oldFiles: FileMeta[], nextFiles: FileMeta[]): FileDiffSummary {
+  const oldByPath = new Map(oldFiles.map((f) => [f.path, f] as const))
+  const nextByPath = new Map(nextFiles.map((f) => [f.path, f] as const))
+
+  const added: string[] = []
+  const removed: string[] = []
+  const changed: string[] = []
+
+  for (const [path, file] of nextByPath.entries()) {
+    const prev = oldByPath.get(path)
+    if (!prev) {
+      added.push(path)
+      continue
+    }
+    if (file.sha256 && prev.sha256 && file.sha256 !== prev.sha256) changed.push(path)
+  }
+  for (const path of oldByPath.keys()) {
+    if (!nextByPath.has(path)) removed.push(path)
+  }
+
+  added.sort()
+  removed.sort()
+  changed.sort()
+  return { added, removed, changed }
+}
+
+function formatDiffSummary(diff: FileDiffSummary) {
+  const parts: string[] = []
+  if (diff.added.length) parts.push(`${diff.added.length} added`)
+  if (diff.changed.length) parts.push(`${diff.changed.length} changed`)
+  if (diff.removed.length) parts.push(`${diff.removed.length} removed`)
+  return parts.join(', ') || 'no file changes detected'
+}
+
+function pickPaths(values: string[]) {
+  if (values.length <= MAX_PATHS_IN_PROMPT) return values
+  return values.slice(0, MAX_PATHS_IN_PROMPT)
+}
+
+function extractResponseText(payload: unknown) {
+  if (!payload || typeof payload !== 'object') return null
+  const output = (payload as { output?: unknown }).output
+  if (!Array.isArray(output)) return null
+  const chunks: string[] = []
+  for (const item of output) {
+    if (!item || typeof item !== 'object') continue
+    if ((item as { type?: unknown }).type !== 'message') continue
+    const content = (item as { content?: unknown }).content
+    if (!Array.isArray(content)) continue
+    for (const part of content) {
+      if (!part || typeof part !== 'object') continue
+      if ((part as { type?: unknown }).type !== 'output_text') continue
+      const text = (part as { text?: unknown }).text
+      if (typeof text === 'string' && text.trim()) chunks.push(text)
+    }
+  }
+  const joined = chunks.join('\n').trim()
+  return joined || null
+}
+
+async function generateWithOpenAI(args: {
+  slug: string
+  version: string
+  oldReadme: string | null
+  nextReadme: string
+  fileDiff: FileDiffSummary | null
+}) {
+  const apiKey = process.env.OPENAI_API_KEY
+  if (!apiKey) return null
+
+  const oldReadme = args.oldReadme ? clampText(args.oldReadme, MAX_README_CHARS) : ''
+  const nextReadme = clampText(args.nextReadme, MAX_README_CHARS)
+
+  const fileDiff = args.fileDiff
+  const diffSummary = fileDiff ? formatDiffSummary(fileDiff) : 'unknown'
+  const changedPaths = fileDiff ? pickPaths(fileDiff.changed) : []
+  const addedPaths = fileDiff ? pickPaths(fileDiff.added) : []
+  const removedPaths = fileDiff ? pickPaths(fileDiff.removed) : []
+
+  const input = [
+    `Soul: ${args.slug}`,
+    `Version: ${args.version}`,
+    `File changes: ${diffSummary}`,
+    changedPaths.length ? `Changed files (sample): ${changedPaths.join(', ')}` : null,
+    addedPaths.length ? `Added files (sample): ${addedPaths.join(', ')}` : null,
+    removedPaths.length ? `Removed files (sample): ${removedPaths.join(', ')}` : null,
+    oldReadme ? `Previous SOUL.md:\n${oldReadme}` : null,
+    `New SOUL.md:\n${nextReadme}`,
+  ]
+    .filter(Boolean)
+    .join('\n\n')
+
+  const response = await fetch('https://api.openai.com/v1/responses', {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: `Bearer ${apiKey}`,
+    },
+    body: JSON.stringify({
+      model: CHANGELOG_MODEL,
+      instructions:
+        'Write a concise changelog for this soul version. Audience: everyone. Output plain text. Prefer 2–6 bullet points. If it is a big change, include a short 1-line summary first, then bullets. Don’t mention that you are AI. Don’t invent details; only use the inputs.',
+      input,
+      max_output_tokens: 220,
+    }),
+  })
+
+  if (!response.ok) return null
+  const payload = (await response.json()) as unknown
+  return extractResponseText(payload)
+}
+
+function generateFallback(args: {
+  slug: string
+  version: string
+  oldReadme: string | null
+  nextReadme: string
+  fileDiff: FileDiffSummary | null
+}) {
+  const lines: string[] = []
+  if (!args.oldReadme) {
+    lines.push(`- Initial release.`)
+    return lines.join('\n')
+  }
+
+  const diff = args.fileDiff
+  if (diff) {
+    const parts: string[] = []
+    if (diff.added.length) parts.push(`added ${diff.added.length}`)
+    if (diff.changed.length) parts.push(`updated ${diff.changed.length}`)
+    if (diff.removed.length) parts.push(`removed ${diff.removed.length}`)
+    if (parts.length) lines.push(`- ${parts.join(', ')} file(s).`)
+  }
+
+  lines.push(`- Updated SOUL.md.`)
+  return lines.join('\n')
+}
+
+export async function generateSoulChangelogForPublish(
+  ctx: ActionCtx,
+  args: { slug: string; version: string; readmeText: string; files: FileMeta[] },
+): Promise<string> {
+  try {
+    const soul = (await ctx.runQuery(internal.souls.getSoulBySlugInternal, {
+      slug: args.slug,
+    })) as Doc<'souls'> | null
+    const previous: Doc<'soulVersions'> | null =
+      soul?.latestVersionId && !soul.softDeletedAt
+        ? ((await ctx.runQuery(internal.souls.getVersionByIdInternal, {
+            versionId: soul.latestVersionId,
+          })) as Doc<'soulVersions'> | null)
+        : null
+
+    const oldReadmeText: string | null = previous
+      ? await readReadmeFromVersion(ctx, previous)
+      : null
+    const oldFiles = previous
+      ? previous.files.map((file) => ({ path: file.path, sha256: file.sha256 }))
+      : []
+    const fileDiff = previous ? summarizeFileDiff(oldFiles, args.files) : null
+
+    const ai = await generateWithOpenAI({
+      slug: args.slug,
+      version: args.version,
+      oldReadme: oldReadmeText,
+      nextReadme: args.readmeText,
+      fileDiff,
+    }).catch(() => null)
+
+    return (
+      ai ??
+      generateFallback({
+        slug: args.slug,
+        version: args.version,
+        oldReadme: oldReadmeText,
+        nextReadme: args.readmeText,
+        fileDiff,
+      })
+    )
+  } catch {
+    return '- Updated soul.'
+  }
+}
+
+export async function generateSoulChangelogPreview(
+  ctx: ActionCtx,
+  args: {
+    slug: string
+    version: string
+    readmeText: string
+    filePaths?: string[]
+  },
+): Promise<string> {
+  try {
+    const soul = (await ctx.runQuery(internal.souls.getSoulBySlugInternal, {
+      slug: args.slug,
+    })) as Doc<'souls'> | null
+    const previous: Doc<'soulVersions'> | null =
+      soul?.latestVersionId && !soul.softDeletedAt
+        ? ((await ctx.runQuery(internal.souls.getVersionByIdInternal, {
+            versionId: soul.latestVersionId,
+          })) as Doc<'soulVersions'> | null)
+        : null
+
+    const oldReadmeText: string | null = previous
+      ? await readReadmeFromVersion(ctx, previous)
+      : null
+    const oldPaths = previous ? previous.files.map((file) => file.path) : []
+    const nextPaths = args.filePaths ?? []
+    const diff = previous ? summarizeFileDiffFromPaths(oldPaths, nextPaths) : null
+
+    const ai = await generateWithOpenAI({
+      slug: args.slug,
+      version: args.version,
+      oldReadme: oldReadmeText,
+      nextReadme: args.readmeText,
+      fileDiff: diff,
+    }).catch(() => null)
+
+    return (
+      ai ??
+      generateFallback({
+        slug: args.slug,
+        version: args.version,
+        oldReadme: oldReadmeText,
+        nextReadme: args.readmeText,
+        fileDiff: diff,
+      })
+    )
+  } catch {
+    return '- Updated soul.'
+  }
+}
+
+async function readReadmeFromVersion(ctx: ActionCtx, version: Doc<'soulVersions'>) {
+  const file = version.files.find((entry) => entry.path.toLowerCase() === 'soul.md')
+  if (!file) return null
+  const blob = await ctx.storage.get(file.storageId)
+  if (!blob) return null
+  return blob.text()
+}
+
+function summarizeFileDiffFromPaths(oldPaths: string[], nextPaths: string[]) {
+  const oldFiles = oldPaths.map((path) => ({ path }))
+  const nextFiles = nextPaths.map((path) => ({ path }))
+  return summarizeFileDiff(oldFiles, nextFiles)
+}
+
+export const __test = {
+  summarizeFileDiff,
+}

package/convex/lib/soulPublish.ts

@@ -0,0 +1,236 @@
+import { ConvexError } from 'convex/values'
+import semver from 'semver'
+import { internal } from '../_generated/api'
+import type { Doc, Id } from '../_generated/dataModel'
+import type { ActionCtx } from '../_generated/server'
+import { generateEmbedding } from './embeddings'
+import {
+  buildEmbeddingText,
+  getFrontmatterMetadata,
+  getFrontmatterValue,
+  hashSkillFiles,
+  isTextFile,
+  parseFrontmatter,
+  sanitizePath,
+} from './skills'
+import { generateSoulChangelogForPublish } from './soulChangelog'
+
+const MAX_TOTAL_BYTES = 50 * 1024 * 1024
+
+const MAX_SUMMARY_LENGTH = 160
+
+function deriveSoulSummary(readmeText: string) {
+  const lines = readmeText.split(/\r?\n/)
+  let inFrontmatter = false
+  for (const raw of lines) {
+    const trimmed = raw.trim()
+    if (!trimmed) continue
+    if (!inFrontmatter && trimmed === '---') {
+      inFrontmatter = true
+      continue
+    }
+    if (inFrontmatter) {
+      if (trimmed === '---') {
+        inFrontmatter = false
+      }
+      continue
+    }
+    const cleaned = trimmed.replace(/^#+\s*/, '')
+    if (!cleaned) continue
+    if (cleaned.length > MAX_SUMMARY_LENGTH) {
+      return `${cleaned.slice(0, MAX_SUMMARY_LENGTH - 3).trimEnd()}...`
+    }
+    return cleaned
+  }
+  return undefined
+}
+
+export type PublishResult = {
+  soulId: Id<'souls'>
+  versionId: Id<'soulVersions'>
+  embeddingId: Id<'soulEmbeddings'>
+}
+
+export type PublishVersionArgs = {
+  slug: string
+  displayName: string
+  version: string
+  changelog: string
+  tags?: string[]
+  source?: {
+    kind: 'github'
+    url: string
+    repo: string
+    ref: string
+    commit: string
+    path: string
+    importedAt: number
+  }
+  files: Array<{
+    path: string
+    size: number
+    storageId: Id<'_storage'>
+    sha256: string
+    contentType?: string
+  }>
+}
+
+export async function publishSoulVersionForUser(
+  ctx: ActionCtx,
+  userId: Id<'users'>,
+  args: PublishVersionArgs,
+): Promise<PublishResult> {
+  const version = args.version.trim()
+  const slug = args.slug.trim().toLowerCase()
+  const displayName = args.displayName.trim()
+  if (!slug || !displayName) throw new ConvexError('Slug and display name required')
+  if (!/^[a-z0-9][a-z0-9-]*$/.test(slug)) {
+    throw new ConvexError('Slug must be lowercase and url-safe')
+  }
+  if (!semver.valid(version)) {
+    throw new ConvexError('Version must be valid semver')
+  }
+  const suppliedChangelog = args.changelog.trim()
+  const changelogSource = suppliedChangelog ? ('user' as const) : ('auto' as const)
+
+  const sanitizedFiles = args.files.map((file) => {
+    const path = sanitizePath(file.path)
+    if (!path) throw new ConvexError('Invalid file paths')
+    if (!isTextFile(path, file.contentType ?? undefined)) {
+      throw new ConvexError('Only text-based files are allowed')
+    }
+    return { ...file, path }
+  })
+
+  const totalBytes = sanitizedFiles.reduce((sum, file) => sum + file.size, 0)
+  if (totalBytes > MAX_TOTAL_BYTES) {
+    throw new ConvexError('Soul bundle exceeds 50MB limit')
+  }
+
+  const isSoulFile = (path: string) => path.toLowerCase() === 'soul.md'
+  const readmeFile = sanitizedFiles.find((file) => isSoulFile(file.path))
+  if (!readmeFile) throw new ConvexError('SOUL.md is required')
+
+  const nonSoulFiles = sanitizedFiles.filter((file) => !isSoulFile(file.path))
+  if (nonSoulFiles.length > 0) {
+    throw new ConvexError('Only SOUL.md is allowed for soul bundles')
+  }
+
+  const readmeText = await fetchText(ctx, readmeFile.storageId)
+  const frontmatter = parseFrontmatter(readmeText)
+  const summary = getFrontmatterValue(frontmatter, 'description') ?? deriveSoulSummary(readmeText)
+  const metadata = mergeSourceIntoMetadata(getFrontmatterMetadata(frontmatter), args.source)
+
+  const embeddingText = buildEmbeddingText({
+    frontmatter,
+    readme: readmeText,
+    otherFiles: [],
+  })
+
+  const fingerprint = await hashSkillFiles(
+    sanitizedFiles.map((file) => ({
+      path: file.path ?? '',
+      sha256: file.sha256,
+    })),
+  )
+
+  const changelogPromise =
+    changelogSource === 'user'
+      ? Promise.resolve(suppliedChangelog)
+      : generateSoulChangelogForPublish(ctx, {
+          slug,
+          version,
+          readmeText,
+          files: sanitizedFiles.map((file) => ({ path: file.path ?? '', sha256: file.sha256 })),
+        })
+
+  const embeddingPromise = generateEmbedding(embeddingText)
+
+  const [changelogText, embedding] = await Promise.all([
+    changelogPromise,
+    embeddingPromise.catch((error) => {
+      throw new ConvexError(formatEmbeddingError(error))
+    }),
+  ])
+
+  const publishResult = (await ctx.runMutation(internal.souls.insertVersion, {
+    userId,
+    slug,
+    displayName,
+    version,
+    changelog: changelogText,
+    changelogSource,
+    tags: args.tags?.map((tag) => tag.trim()).filter(Boolean),
+    fingerprint,
+    files: sanitizedFiles,
+    parsed: {
+      frontmatter,
+      metadata,
+    },
+    summary,
+    embedding,
+  })) as PublishResult
+
+  const owner = (await ctx.runQuery(internal.users.getByIdInternal, {
+    userId,
+  })) as Doc<'users'> | null
+  const ownerHandle = owner?.handle ?? owner?.name ?? userId
+
+  void ctx.scheduler
+    .runAfter(0, internal.githubSoulBackupsNode.backupSoulForPublishInternal, {
+      slug,
+      version,
+      displayName,
+      ownerHandle,
+      files: sanitizedFiles,
+      publishedAt: Date.now(),
+    })
+    .catch((error) => {
+      console.error('GitHub soul backup scheduling failed', error)
+    })
+
+  return publishResult
+}
+
+function mergeSourceIntoMetadata(metadata: unknown, source: PublishVersionArgs['source']) {
+  if (!source) return metadata === undefined ? undefined : metadata
+  const sourceValue = {
+    kind: source.kind,
+    url: source.url,
+    repo: source.repo,
+    ref: source.ref,
+    commit: source.commit,
+    path: source.path,
+    importedAt: source.importedAt,
+  }
+
+  if (!metadata) return { source: sourceValue }
+  if (typeof metadata !== 'object' || Array.isArray(metadata)) return { source: sourceValue }
+  return { ...(metadata as Record<string, unknown>), source: sourceValue }
+}
+
+export async function fetchText(
+  ctx: { storage: { get: (id: Id<'_storage'>) => Promise<Blob | null> } },
+  storageId: Id<'_storage'>,
+) {
+  const blob = await ctx.storage.get(storageId)
+  if (!blob) throw new Error('File missing in storage')
+  return blob.text()
+}
+
+function formatEmbeddingError(error: unknown) {
+  if (error instanceof Error) {
+    if (error.message.includes('OPENAI_API_KEY')) {
+      return 'OPENAI_API_KEY is not configured.'
+    }
+    if (error.message.startsWith('Embedding failed')) {
+      return error.message
+    }
+  }
+  return 'Embedding failed. Please try again.'
+}
+
+export const __test = {
+  getSummary: (frontmatter: Record<string, unknown>) =>
+    getFrontmatterValue(frontmatter, 'description'),
+}

package/convex/lib/tokens.test.ts

@@ -0,0 +1,33 @@
+/* @vitest-environment node */
+
+import { describe, expect, it } from 'vitest'
+import { __test, generateToken, hashToken } from './tokens'
+
+describe('tokens', () => {
+  it('hashToken returns sha256 hex', async () => {
+    await expect(hashToken('test')).resolves.toBe(
+      '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08',
+    )
+  })
+
+  it('generateToken returns token + prefix', () => {
+    const { token, prefix } = generateToken()
+    expect(token).toMatch(/^clh_[A-Za-z0-9_-]+$/)
+    expect(prefix).toBe(token.slice(0, 12))
+  })
+
+  it('toHex encodes bytes', () => {
+    expect(__test.toHex(new Uint8Array([0, 15, 255]))).toBe('000fff')
+  })
+
+  it('toBase64 encodes 1/2/3-byte tails', () => {
+    expect(__test.toBase64(new Uint8Array([0xff]))).toBe('/w==')
+    expect(__test.toBase64(new Uint8Array([0xff, 0xee]))).toBe('/+4=')
+    expect(__test.toBase64(new Uint8Array([0xff, 0xee, 0xdd]))).toBe('/+7d')
+  })
+
+  it('toBase64Url replaces alphabet and strips padding', () => {
+    expect(__test.toBase64Url(new Uint8Array([0xff]))).toBe('_w')
+    expect(__test.toBase64Url(new Uint8Array([0xfa, 0x00, 0x00]))).toBe('-gAA')
+  })
+})

package/convex/lib/tokens.ts

@@ -0,0 +1,51 @@
+const encoder = new TextEncoder()
+
+export const API_TOKEN_PREFIX = 'clh_'
+
+export async function hashToken(token: string) {
+  const bytes = encoder.encode(token)
+  const digest = await crypto.subtle.digest('SHA-256', bytes)
+  return toHex(new Uint8Array(digest))
+}
+
+export function generateToken() {
+  const bytes = new Uint8Array(32)
+  crypto.getRandomValues(bytes)
+  const token = `${API_TOKEN_PREFIX}${toBase64Url(bytes)}`
+  const prefix = token.slice(0, 12)
+  return { token, prefix }
+}
+
+function toHex(bytes: Uint8Array) {
+  let out = ''
+  for (const byte of bytes) out += byte.toString(16).padStart(2, '0')
+  return out
+}
+
+function toBase64Url(bytes: Uint8Array) {
+  const base64 = toBase64(bytes)
+  return base64.replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/g, '')
+}
+
+const BASE64_ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
+
+function toBase64(bytes: Uint8Array) {
+  let output = ''
+  for (let i = 0; i < bytes.length; i += 3) {
+    const a = bytes[i] ?? 0
+    const b = bytes[i + 1] ?? 0
+    const c = bytes[i + 2] ?? 0
+    const triple = (a << 16) | (b << 8) | c
+    output += BASE64_ALPHABET[(triple >> 18) & 63]
+    output += BASE64_ALPHABET[(triple >> 12) & 63]
+    output += i + 1 < bytes.length ? BASE64_ALPHABET[(triple >> 6) & 63] : '='
+    output += i + 2 < bytes.length ? BASE64_ALPHABET[triple & 63] : '='
+  }
+  return output
+}
+
+export const __test = {
+  toHex,
+  toBase64,
+  toBase64Url,
+}

package/convex/lib/webhooks.test.ts

@@ -0,0 +1,91 @@
+/* @vitest-environment node */
+import { afterEach, describe, expect, it } from 'vitest'
+import { buildDiscordPayload, buildSkillUrl, getWebhookConfig, shouldSendWebhook } from './webhooks'
+
+const originalEnv = { ...process.env }
+
+afterEach(() => {
+  process.env = { ...originalEnv }
+})
+
+describe('webhook config', () => {
+  it('parses highlighted-only flag', () => {
+    process.env.DISCORD_WEBHOOK_URL = 'https://example.com'
+    process.env.DISCORD_WEBHOOK_HIGHLIGHTED_ONLY = 'true'
+    const config = getWebhookConfig()
+    expect(config.highlightedOnly).toBe(true)
+  })
+
+  it('defaults site url when missing', () => {
+    delete process.env.SITE_URL
+    process.env.DISCORD_WEBHOOK_URL = 'https://example.com'
+    const config = getWebhookConfig()
+    expect(config.siteUrl).toBe('https://pilothub.com')
+  })
+})
+
+describe('webhook filtering', () => {
+  it('skips when url missing', () => {
+    const config = getWebhookConfig({} as NodeJS.ProcessEnv)
+    expect(shouldSendWebhook('skill.publish', { slug: 'demo', displayName: 'Demo' }, config)).toBe(
+      false,
+    )
+  })
+
+  it('filters non-highlighted when highlighted-only', () => {
+    const config = {
+      url: 'https://example.com',
+      highlightedOnly: true,
+      siteUrl: 'https://pilothub.com',
+    }
+    const allowed = shouldSendWebhook(
+      'skill.publish',
+      { slug: 'demo', displayName: 'Demo', highlighted: false },
+      config,
+    )
+    expect(allowed).toBe(false)
+  })
+
+  it('allows highlighted event when highlighted-only', () => {
+    const config = {
+      url: 'https://example.com',
+      highlightedOnly: true,
+      siteUrl: 'https://pilothub.com',
+    }
+    const allowed = shouldSendWebhook(
+      'skill.highlighted',
+      { slug: 'demo', displayName: 'Demo', highlighted: true },
+      config,
+    )
+    expect(allowed).toBe(true)
+  })
+})
+
+describe('payload building', () => {
+  it('builds canonical url with owner', () => {
+    const url = buildSkillUrl(
+      { slug: 'beeper', displayName: 'Beeper', ownerHandle: 'KrauseFx' },
+      'https://pilothub.com',
+    )
+    expect(url).toBe('https://pilothub.com/KrauseFx/beeper')
+  })
+
+  it('builds a publish embed', () => {
+    const payload = buildDiscordPayload(
+      'skill.publish',
+      {
+        slug: 'demo',
+        displayName: 'Demo Skill',
+        summary: 'Nice skill',
+        version: '1.2.3',
+        ownerHandle: 'steipete',
+        tags: ['latest', 'discord'],
+      },
+      { url: 'https://example.com', highlightedOnly: false, siteUrl: 'https://pilothub.com' },
+    )
+    const embed = payload.embeds[0]
+    expect(embed.title).toBe('Demo Skill')
+    expect(embed.description).toBe('Nice skill')
+    expect(embed.fields[0].value).toBe('v1.2.3')
+  })
+})