pilothub 0.0.1

package/convex/githubBackups.ts

@@ -0,0 +1,170 @@
+import { v } from 'convex/values'
+import { internal } from './_generated/api'
+import type { Doc, Id } from './_generated/dataModel'
+import { action, internalMutation, internalQuery } from './_generated/server'
+import { assertRole, requireUserFromAction } from './lib/access'
+
+const DEFAULT_BATCH_SIZE = 50
+const MAX_BATCH_SIZE = 200
+const SYNC_STATE_KEY = 'default'
+
+type BackupPageItem =
+  | {
+      kind: 'ok'
+      skillId: Id<'skills'>
+      versionId: Id<'skillVersions'>
+      slug: string
+      displayName: string
+      version: string
+      ownerHandle: string
+      files: Doc<'skillVersions'>['files']
+      publishedAt: number
+    }
+  | { kind: 'missingLatestVersion'; skillId: Id<'skills'> }
+  | { kind: 'missingVersionDoc'; skillId: Id<'skills'>; versionId: Id<'skillVersions'> }
+  | { kind: 'missingOwner'; skillId: Id<'skills'>; ownerUserId: Id<'users'> }
+
+type BackupPageResult = {
+  items: BackupPageItem[]
+  cursor: string | null
+  isDone: boolean
+}
+
+type BackupSyncState = {
+  cursor: string | null
+}
+
+export type SyncGitHubBackupsResult = {
+  stats: {
+    skillsScanned: number
+    skillsSkipped: number
+    skillsBackedUp: number
+    skillsMissingVersion: number
+    skillsMissingOwner: number
+    errors: number
+  }
+  cursor: string | null
+  isDone: boolean
+}
+
+export const getGitHubBackupPageInternal = internalQuery({
+  args: {
+    cursor: v.optional(v.string()),
+    batchSize: v.optional(v.number()),
+  },
+  handler: async (ctx, args): Promise<BackupPageResult> => {
+    const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
+    const { page, isDone, continueCursor } = await ctx.db
+      .query('skills')
+      .order('asc')
+      .paginate({ cursor: args.cursor ?? null, numItems: batchSize })
+
+    const items: BackupPageItem[] = []
+    for (const skill of page) {
+      if (skill.softDeletedAt) continue
+      if (!skill.latestVersionId) {
+        items.push({ kind: 'missingLatestVersion', skillId: skill._id })
+        continue
+      }
+
+      const version = await ctx.db.get(skill.latestVersionId)
+      if (!version) {
+        items.push({
+          kind: 'missingVersionDoc',
+          skillId: skill._id,
+          versionId: skill.latestVersionId,
+        })
+        continue
+      }
+
+      const owner = await ctx.db.get(skill.ownerUserId)
+      if (!owner || owner.deletedAt) {
+        items.push({ kind: 'missingOwner', skillId: skill._id, ownerUserId: skill.ownerUserId })
+        continue
+      }
+
+      items.push({
+        kind: 'ok',
+        skillId: skill._id,
+        versionId: version._id,
+        slug: skill.slug,
+        displayName: skill.displayName,
+        version: version.version,
+        ownerHandle: owner.handle ?? owner._id,
+        files: version.files,
+        publishedAt: version.createdAt,
+      })
+    }
+
+    return { items, cursor: continueCursor, isDone }
+  },
+})
+
+export const getGitHubBackupSyncStateInternal = internalQuery({
+  args: {},
+  handler: async (ctx): Promise<BackupSyncState> => {
+    const state = await ctx.db
+      .query('githubBackupSyncState')
+      .withIndex('by_key', (q) => q.eq('key', SYNC_STATE_KEY))
+      .unique()
+    return { cursor: state?.cursor ?? null }
+  },
+})
+
+export const setGitHubBackupSyncStateInternal = internalMutation({
+  args: {
+    cursor: v.optional(v.string()),
+  },
+  handler: async (ctx, args) => {
+    const now = Date.now()
+    const state = await ctx.db
+      .query('githubBackupSyncState')
+      .withIndex('by_key', (q) => q.eq('key', SYNC_STATE_KEY))
+      .unique()
+
+    if (!state) {
+      await ctx.db.insert('githubBackupSyncState', {
+        key: SYNC_STATE_KEY,
+        cursor: args.cursor,
+        updatedAt: now,
+      })
+      return { ok: true as const }
+    }
+
+    await ctx.db.patch(state._id, {
+      cursor: args.cursor,
+      updatedAt: now,
+    })
+
+    return { ok: true as const }
+  },
+})
+
+export const syncGitHubBackups: ReturnType<typeof action> = action({
+  args: {
+    dryRun: v.optional(v.boolean()),
+    batchSize: v.optional(v.number()),
+    maxBatches: v.optional(v.number()),
+    resetCursor: v.optional(v.boolean()),
+  },
+  handler: async (ctx, args): Promise<SyncGitHubBackupsResult> => {
+    const { user } = await requireUserFromAction(ctx)
+    assertRole(user, ['admin'])
+
+    if (args.resetCursor && !args.dryRun) {
+      await ctx.runMutation(internal.githubBackups.setGitHubBackupSyncStateInternal, {
+        cursor: undefined,
+      })
+    }
+
+    return ctx.runAction(internal.githubBackupsNode.syncGitHubBackupsInternal, {
+      dryRun: args.dryRun,
+      batchSize: args.batchSize,
+      maxBatches: args.maxBatches,
+    }) as Promise<SyncGitHubBackupsResult>
+  },
+})
+
+function clampInt(value: number, min: number, max: number) {
+  return Math.max(min, Math.min(max, Math.floor(value)))
+}

package/convex/githubBackupsNode.ts

@@ -0,0 +1,183 @@
+'use node'
+
+import { v } from 'convex/values'
+import { internal } from './_generated/api'
+import type { Doc } from './_generated/dataModel'
+import type { ActionCtx } from './_generated/server'
+import { internalAction } from './_generated/server'
+import {
+  backupSkillToGitHub,
+  fetchGitHubSkillMeta,
+  getGitHubBackupContext,
+  isGitHubBackupConfigured,
+} from './lib/githubBackup'
+
+const DEFAULT_BATCH_SIZE = 50
+const MAX_BATCH_SIZE = 200
+const DEFAULT_MAX_BATCHES = 5
+const MAX_MAX_BATCHES = 200
+
+type BackupPageItem =
+  | {
+      kind: 'ok'
+      slug: string
+      version: string
+      displayName: string
+      ownerHandle: string
+      files: Doc<'skillVersions'>['files']
+      publishedAt: number
+    }
+  | { kind: 'missingLatestVersion' }
+  | { kind: 'missingVersionDoc' }
+  | { kind: 'missingOwner' }
+
+export type GitHubBackupSyncStats = {
+  skillsScanned: number
+  skillsSkipped: number
+  skillsBackedUp: number
+  skillsMissingVersion: number
+  skillsMissingOwner: number
+  errors: number
+}
+
+export type SyncGitHubBackupsInternalArgs = {
+  dryRun?: boolean
+  batchSize?: number
+  maxBatches?: number
+}
+
+export type SyncGitHubBackupsInternalResult = {
+  stats: GitHubBackupSyncStats
+  cursor: string | null
+  isDone: boolean
+}
+
+export const backupSkillForPublishInternal = internalAction({
+  args: {
+    slug: v.string(),
+    version: v.string(),
+    displayName: v.string(),
+    ownerHandle: v.string(),
+    files: v.array(
+      v.object({
+        path: v.string(),
+        size: v.number(),
+        storageId: v.id('_storage'),
+        sha256: v.string(),
+        contentType: v.optional(v.string()),
+      }),
+    ),
+    publishedAt: v.number(),
+  },
+  handler: async (ctx, args) => {
+    if (!isGitHubBackupConfigured()) {
+      return { skipped: true as const }
+    }
+    await backupSkillToGitHub(ctx, args)
+    return { skipped: false as const }
+  },
+})
+
+export async function syncGitHubBackupsInternalHandler(
+  ctx: ActionCtx,
+  args: SyncGitHubBackupsInternalArgs,
+): Promise<SyncGitHubBackupsInternalResult> {
+  const dryRun = Boolean(args.dryRun)
+  const stats: GitHubBackupSyncStats = {
+    skillsScanned: 0,
+    skillsSkipped: 0,
+    skillsBackedUp: 0,
+    skillsMissingVersion: 0,
+    skillsMissingOwner: 0,
+    errors: 0,
+  }
+
+  if (!isGitHubBackupConfigured()) {
+    return { stats, cursor: null, isDone: true }
+  }
+
+  const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
+  const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES)
+  const context = await getGitHubBackupContext()
+
+  const state = dryRun
+    ? { cursor: null as string | null }
+    : ((await ctx.runQuery(internal.githubBackups.getGitHubBackupSyncStateInternal, {})) as {
+        cursor: string | null
+      })
+
+  let cursor: string | null = state.cursor
+  let isDone = false
+
+  for (let batch = 0; batch < maxBatches; batch++) {
+    const page = (await ctx.runQuery(internal.githubBackups.getGitHubBackupPageInternal, {
+      cursor: cursor ?? undefined,
+      batchSize,
+    })) as { items: BackupPageItem[]; cursor: string | null; isDone: boolean }
+
+    cursor = page.cursor
+    isDone = page.isDone
+
+    for (const item of page.items) {
+      if (item.kind !== 'ok') {
+        if (item.kind === 'missingLatestVersion' || item.kind === 'missingVersionDoc') {
+          stats.skillsMissingVersion += 1
+        } else if (item.kind === 'missingOwner') {
+          stats.skillsMissingOwner += 1
+        }
+        continue
+      }
+
+      stats.skillsScanned += 1
+      try {
+        const meta = await fetchGitHubSkillMeta(context, item.ownerHandle, item.slug)
+        if (meta?.latest?.version === item.version) {
+          stats.skillsSkipped += 1
+          continue
+        }
+
+        if (!dryRun) {
+          await backupSkillToGitHub(
+            ctx,
+            {
+              slug: item.slug,
+              version: item.version,
+              displayName: item.displayName,
+              ownerHandle: item.ownerHandle,
+              files: item.files,
+              publishedAt: item.publishedAt,
+            },
+            context,
+          )
+          stats.skillsBackedUp += 1
+        }
+      } catch (error) {
+        console.error('GitHub backup sync failed', error)
+        stats.errors += 1
+      }
+    }
+
+    if (!dryRun) {
+      await ctx.runMutation(internal.githubBackups.setGitHubBackupSyncStateInternal, {
+        cursor: isDone ? undefined : (cursor ?? undefined),
+      })
+    }
+
+    if (isDone) break
+  }
+
+  return { stats, cursor, isDone }
+}
+
+export const syncGitHubBackupsInternal = internalAction({
+  args: {
+    dryRun: v.optional(v.boolean()),
+    batchSize: v.optional(v.number()),
+    maxBatches: v.optional(v.number()),
+  },
+  handler: syncGitHubBackupsInternalHandler,
+})
+
+function clampInt(value: number, min: number, max: number) {
+  return Math.max(min, Math.min(max, Math.floor(value)))
+}

package/convex/githubImport.ts

@@ -0,0 +1,317 @@
+import { ConvexError, v } from 'convex/values'
+import { unzipSync } from 'fflate'
+import semver from 'semver'
+import { api, internal } from './_generated/api'
+import type { Id } from './_generated/dataModel'
+import type { ActionCtx } from './_generated/server'
+import { action } from './_generated/server'
+import { requireUserFromAction } from './lib/access'
+import {
+  buildGitHubImportFileList,
+  computeDefaultSelectedPaths,
+  detectGitHubImportCandidates,
+  fetchGitHubZipBytes,
+  listTextFilesUnderCandidate,
+  normalizeRepoPath,
+  parseGitHubImportUrl,
+  resolveGitHubCommit,
+  stripGitHubZipRoot,
+  suggestDisplayName,
+  suggestVersion,
+} from './lib/githubImport'
+import { publishVersionForUser } from './lib/skillPublish'
+import { sanitizePath } from './lib/skills'
+
+const MAX_SELECTED_BYTES = 50 * 1024 * 1024
+const MAX_UNZIPPED_BYTES = 80 * 1024 * 1024
+const MAX_FILE_COUNT = 7_500
+const MAX_SINGLE_FILE_BYTES = 10 * 1024 * 1024
+
+export const previewGitHubImport = action({
+  args: { url: v.string() },
+  handler: async (ctx, args) => {
+    await requireUserFromAction(ctx)
+
+    const parsed = parseGitHubImportUrl(args.url)
+    const resolved = await resolveGitHubCommit(parsed, fetch)
+    const zipBytes = await fetchGitHubZipBytes(resolved, fetch)
+    const entries = unzipToEntries(zipBytes)
+    const stripped = stripGitHubZipRoot(entries)
+    const candidates = detectGitHubImportCandidates(stripped).filter((candidate) =>
+      isCandidateUnderResolvedPath(candidate.path, resolved.path),
+    )
+    if (candidates.length === 0) throw new ConvexError('No SKILL.md found in this repo')
+
+    return {
+      resolved,
+      candidates: candidates.map((candidate) => ({
+        path: candidate.path,
+        readmePath: candidate.readmePath,
+        name: candidate.name ?? null,
+        description: candidate.description ?? null,
+      })),
+    }
+  },
+})
+
+export const previewGitHubImportCandidate = action({
+  args: { url: v.string(), candidatePath: v.string() },
+  handler: async (ctx, args) => {
+    const { userId } = await requireUserFromAction(ctx)
+
+    const parsed = parseGitHubImportUrl(args.url)
+    const resolved = await resolveGitHubCommit(parsed, fetch)
+    const zipBytes = await fetchGitHubZipBytes(resolved, fetch)
+    const entries = unzipToEntries(zipBytes)
+    const stripped = stripGitHubZipRoot(entries)
+
+    const normalizedCandidatePath = normalizeRepoPath(args.candidatePath)
+    if (!isCandidateUnderResolvedPath(normalizedCandidatePath, resolved.path)) {
+      throw new ConvexError('Candidate path is outside the requested import scope')
+    }
+
+    const candidates = detectGitHubImportCandidates(stripped).filter((candidate) =>
+      isCandidateUnderResolvedPath(candidate.path, resolved.path),
+    )
+
+    const candidate = candidates.find((item) => item.path === normalizedCandidatePath)
+    if (!candidate) throw new ConvexError('Candidate not found')
+
+    const files = listTextFilesUnderCandidate(stripped, candidate.path)
+    const defaultSelectedPaths = computeDefaultSelectedPaths({ candidate, files })
+    const fileList = buildGitHubImportFileList({
+      candidate,
+      files,
+      defaultSelectedPaths,
+    })
+
+    const baseForNaming = candidate.path ? (candidate.path.split('/').at(-1) ?? '') : resolved.repo
+    const suggestedDisplayName = suggestDisplayName(candidate, baseForNaming)
+
+    const rawSlugBase = sanitizeSlug(candidate.path ? baseForNaming : resolved.repo)
+    const suggestedSlug = await suggestAvailableSlug(ctx, userId, rawSlugBase)
+
+    const existing = await ctx.runQuery(api.skills.getBySlug, { slug: suggestedSlug })
+    const existingLatest =
+      existing?.skill && existing.skill.ownerUserId === userId
+        ? (existing.latestVersion?.version ?? null)
+        : null
+    const suggestedVersion = suggestVersion(existingLatest)
+
+    return {
+      resolved,
+      candidate: {
+        path: candidate.path,
+        readmePath: candidate.readmePath,
+        name: candidate.name ?? null,
+        description: candidate.description ?? null,
+      },
+      defaults: {
+        selectedPaths: defaultSelectedPaths,
+        slug: suggestedSlug,
+        displayName: suggestedDisplayName,
+        version: suggestedVersion,
+        tags: ['latest'],
+      },
+      files: fileList,
+    }
+  },
+})
+
+export const importGitHubSkill = action({
+  args: {
+    url: v.string(),
+    commit: v.string(),
+    candidatePath: v.string(),
+    selectedPaths: v.array(v.string()),
+    slug: v.optional(v.string()),
+    displayName: v.optional(v.string()),
+    version: v.optional(v.string()),
+    tags: v.optional(v.array(v.string())),
+  },
+  handler: async (ctx, args) => {
+    const { userId } = await requireUserFromAction(ctx)
+
+    const parsed = parseGitHubImportUrl(args.url)
+    const resolved = await resolveGitHubCommit(parsed, fetch)
+    if (!/^[a-f0-9]{40}$/i.test(args.commit)) throw new ConvexError('Invalid commit')
+    if (args.commit.toLowerCase() !== resolved.commit.toLowerCase()) {
+      throw new ConvexError('Import is out of date. Re-run preview.')
+    }
+
+    const normalizedCandidatePath = normalizeRepoPath(args.candidatePath)
+    if (!isCandidateUnderResolvedPath(normalizedCandidatePath, resolved.path)) {
+      throw new ConvexError('Candidate path is outside the requested import scope')
+    }
+
+    const zipBytes = await fetchGitHubZipBytes(resolved, fetch)
+    const entries = stripGitHubZipRoot(unzipToEntries(zipBytes))
+
+    const candidates = detectGitHubImportCandidates(entries).filter((candidate) =>
+      isCandidateUnderResolvedPath(candidate.path, resolved.path),
+    )
+    const candidate = candidates.find((item) => item.path === normalizedCandidatePath)
+    if (!candidate) throw new ConvexError('Candidate not found')
+
+    const filesUnderCandidate = listTextFilesUnderCandidate(entries, candidate.path)
+    const byPath = new Map(filesUnderCandidate.map((file) => [file.path, file.bytes]))
+
+    const selected = Array.from(
+      new Set(args.selectedPaths.map((path) => normalizeRepoPath(path)).filter(Boolean)),
+    )
+    if (selected.length === 0) throw new ConvexError('No files selected')
+
+    const candidateRoot = candidate.path ? `${candidate.path}/` : ''
+    const normalizedReadmePath = normalizeRepoPath(candidate.readmePath)
+    if (!selected.includes(normalizedReadmePath)) {
+      throw new ConvexError('SKILL.md must be selected')
+    }
+
+    let totalBytes = 0
+    const storedFiles: Array<{
+      path: string
+      size: number
+      storageId: Id<'_storage'>
+      sha256: string
+      contentType?: string
+    }> = []
+
+    for (const path of selected.sort()) {
+      if (candidateRoot && !path.startsWith(candidateRoot)) {
+        throw new ConvexError('Selected file is outside the chosen skill folder')
+      }
+
+      const bytes = byPath.get(path)
+      if (!bytes) continue
+      totalBytes += bytes.byteLength
+      if (totalBytes > MAX_SELECTED_BYTES) throw new ConvexError('Selected files exceed 50MB limit')
+
+      const relPath = candidateRoot ? path.slice(candidateRoot.length) : path
+      const sanitized = sanitizePath(relPath)
+      if (!sanitized) throw new ConvexError('Invalid file paths')
+
+      const sha256 = await sha256Hex(bytes)
+      const safeBytes = new Uint8Array(bytes)
+      const storageId = await ctx.storage.store(new Blob([safeBytes], { type: 'text/plain' }))
+      storedFiles.push({
+        path: sanitized,
+        size: bytes.byteLength,
+        storageId,
+        sha256,
+        contentType: 'text/plain',
+      })
+    }
+
+    if (storedFiles.length === 0) throw new ConvexError('No files selected')
+
+    const slugBase = (args.slug ?? '').trim().toLowerCase()
+    const displayName = (args.displayName ?? '').trim()
+    const tags = (args.tags ?? ['latest']).map((tag) => tag.trim()).filter(Boolean)
+    const version = (args.version ?? '').trim()
+
+    if (!slugBase) throw new ConvexError('Slug required')
+    if (!displayName) throw new ConvexError('Display name required')
+    if (!version || !semver.valid(version)) throw new ConvexError('Version must be valid semver')
+
+    const result = await publishVersionForUser(ctx, userId, {
+      slug: slugBase,
+      displayName,
+      version,
+      changelog: '',
+      tags,
+      files: storedFiles,
+      source: {
+        kind: 'github',
+        url: resolved.originalUrl,
+        repo: `${resolved.owner}/${resolved.repo}`,
+        ref: resolved.ref,
+        commit: resolved.commit,
+        path: candidate.path,
+        importedAt: Date.now(),
+      },
+    })
+
+    return { ok: true, slug: slugBase, version, ...result }
+  },
+})
+
+function unzipToEntries(zipBytes: Uint8Array) {
+  const entries = unzipSync(zipBytes)
+  const out: Record<string, Uint8Array> = {}
+  const rawPaths = Object.keys(entries)
+  if (rawPaths.length > MAX_FILE_COUNT) throw new ConvexError('Repo archive has too many files')
+  let totalBytes = 0
+  for (const [rawPath, bytes] of Object.entries(entries)) {
+    const normalizedPath = normalizeZipPath(rawPath)
+    if (!normalizedPath) continue
+    if (isJunkPath(normalizedPath)) continue
+    if (!bytes) continue
+    if (bytes.byteLength > MAX_SINGLE_FILE_BYTES) continue
+    totalBytes += bytes.byteLength
+    if (totalBytes > MAX_UNZIPPED_BYTES) throw new ConvexError('Repo archive is too large')
+    out[normalizedPath] = bytes
+  }
+  return out
+}
+
+function isCandidateUnderResolvedPath(candidatePath: string, resolvedPath: string) {
+  const root = normalizeRepoPath(resolvedPath)
+  if (!root) return true
+  if (!candidatePath) return false
+  if (candidatePath === root) return true
+  return candidatePath.startsWith(`${root}/`)
+}
+
+function sanitizeSlug(value: string) {
+  return value
+    .trim()
+    .toLowerCase()
+    .replace(/[^a-z0-9-]+/g, '-')
+    .replace(/^-+/, '')
+    .replace(/-+$/, '')
+    .replace(/--+/g, '-')
+}
+
+async function suggestAvailableSlug(ctx: ActionCtx, userId: Id<'users'>, base: string) {
+  const cleaned = sanitizeSlug(base)
+  if (!cleaned) throw new ConvexError('Could not derive slug')
+  for (let i = 0; i < 50; i += 1) {
+    const candidate = i === 0 ? cleaned : `${cleaned}-${i + 1}`
+    const existing = await ctx.runQuery(internal.skills.getSkillBySlugInternal, { slug: candidate })
+    if (!existing) return candidate
+    if (existing.ownerUserId === userId) return candidate
+  }
+  throw new ConvexError('Could not find an available slug')
+}
+
+async function sha256Hex(bytes: Uint8Array) {
+  const normalized = new Uint8Array(bytes)
+  const digest = await crypto.subtle.digest('SHA-256', normalized.buffer)
+  return toHex(new Uint8Array(digest))
+}
+
+function toHex(bytes: Uint8Array) {
+  let out = ''
+  for (const byte of bytes) out += byte.toString(16).padStart(2, '0')
+  return out
+}
+
+function normalizeZipPath(path: string) {
+  const normalized = path
+    .replaceAll('\u0000', '')
+    .replaceAll('\\', '/')
+    .trim()
+    .replace(/^\.\/+/, '')
+    .replace(/^\/+/, '')
+  if (!normalized) return ''
+  if (normalized.includes('..')) return ''
+  return normalized
+}
+
+function isJunkPath(path: string) {
+  const normalized = path.toLowerCase()
+  if (normalized.startsWith('__macosx/')) return true
+  if (normalized.endsWith('/.ds_store')) return true
+  if (normalized === '.ds_store') return true
+  return false
+}