pilothub 0.0.1 → 0.0.2

package/convex/githubBackups.ts

@@ -1,170 +0,0 @@
-import { v } from 'convex/values'
-import { internal } from './_generated/api'
-import type { Doc, Id } from './_generated/dataModel'
-import { action, internalMutation, internalQuery } from './_generated/server'
-import { assertRole, requireUserFromAction } from './lib/access'
-
-const DEFAULT_BATCH_SIZE = 50
-const MAX_BATCH_SIZE = 200
-const SYNC_STATE_KEY = 'default'
-
-type BackupPageItem =
-  | {
-      kind: 'ok'
-      skillId: Id<'skills'>
-      versionId: Id<'skillVersions'>
-      slug: string
-      displayName: string
-      version: string
-      ownerHandle: string
-      files: Doc<'skillVersions'>['files']
-      publishedAt: number
-    }
-  | { kind: 'missingLatestVersion'; skillId: Id<'skills'> }
-  | { kind: 'missingVersionDoc'; skillId: Id<'skills'>; versionId: Id<'skillVersions'> }
-  | { kind: 'missingOwner'; skillId: Id<'skills'>; ownerUserId: Id<'users'> }
-
-type BackupPageResult = {
-  items: BackupPageItem[]
-  cursor: string | null
-  isDone: boolean
-}
-
-type BackupSyncState = {
-  cursor: string | null
-}
-
-export type SyncGitHubBackupsResult = {
-  stats: {
-    skillsScanned: number
-    skillsSkipped: number
-    skillsBackedUp: number
-    skillsMissingVersion: number
-    skillsMissingOwner: number
-    errors: number
-  }
-  cursor: string | null
-  isDone: boolean
-}
-
-export const getGitHubBackupPageInternal = internalQuery({
-  args: {
-    cursor: v.optional(v.string()),
-    batchSize: v.optional(v.number()),
-  },
-  handler: async (ctx, args): Promise<BackupPageResult> => {
-    const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
-    const { page, isDone, continueCursor } = await ctx.db
-      .query('skills')
-      .order('asc')
-      .paginate({ cursor: args.cursor ?? null, numItems: batchSize })
-
-    const items: BackupPageItem[] = []
-    for (const skill of page) {
-      if (skill.softDeletedAt) continue
-      if (!skill.latestVersionId) {
-        items.push({ kind: 'missingLatestVersion', skillId: skill._id })
-        continue
-      }
-
-      const version = await ctx.db.get(skill.latestVersionId)
-      if (!version) {
-        items.push({
-          kind: 'missingVersionDoc',
-          skillId: skill._id,
-          versionId: skill.latestVersionId,
-        })
-        continue
-      }
-
-      const owner = await ctx.db.get(skill.ownerUserId)
-      if (!owner || owner.deletedAt) {
-        items.push({ kind: 'missingOwner', skillId: skill._id, ownerUserId: skill.ownerUserId })
-        continue
-      }
-
-      items.push({
-        kind: 'ok',
-        skillId: skill._id,
-        versionId: version._id,
-        slug: skill.slug,
-        displayName: skill.displayName,
-        version: version.version,
-        ownerHandle: owner.handle ?? owner._id,
-        files: version.files,
-        publishedAt: version.createdAt,
-      })
-    }
-
-    return { items, cursor: continueCursor, isDone }
-  },
-})
-
-export const getGitHubBackupSyncStateInternal = internalQuery({
-  args: {},
-  handler: async (ctx): Promise<BackupSyncState> => {
-    const state = await ctx.db
-      .query('githubBackupSyncState')
-      .withIndex('by_key', (q) => q.eq('key', SYNC_STATE_KEY))
-      .unique()
-    return { cursor: state?.cursor ?? null }
-  },
-})
-
-export const setGitHubBackupSyncStateInternal = internalMutation({
-  args: {
-    cursor: v.optional(v.string()),
-  },
-  handler: async (ctx, args) => {
-    const now = Date.now()
-    const state = await ctx.db
-      .query('githubBackupSyncState')
-      .withIndex('by_key', (q) => q.eq('key', SYNC_STATE_KEY))
-      .unique()
-
-    if (!state) {
-      await ctx.db.insert('githubBackupSyncState', {
-        key: SYNC_STATE_KEY,
-        cursor: args.cursor,
-        updatedAt: now,
-      })
-      return { ok: true as const }
-    }
-
-    await ctx.db.patch(state._id, {
-      cursor: args.cursor,
-      updatedAt: now,
-    })
-
-    return { ok: true as const }
-  },
-})
-
-export const syncGitHubBackups: ReturnType<typeof action> = action({
-  args: {
-    dryRun: v.optional(v.boolean()),
-    batchSize: v.optional(v.number()),
-    maxBatches: v.optional(v.number()),
-    resetCursor: v.optional(v.boolean()),
-  },
-  handler: async (ctx, args): Promise<SyncGitHubBackupsResult> => {
-    const { user } = await requireUserFromAction(ctx)
-    assertRole(user, ['admin'])
-
-    if (args.resetCursor && !args.dryRun) {
-      await ctx.runMutation(internal.githubBackups.setGitHubBackupSyncStateInternal, {
-        cursor: undefined,
-      })
-    }
-
-    return ctx.runAction(internal.githubBackupsNode.syncGitHubBackupsInternal, {
-      dryRun: args.dryRun,
-      batchSize: args.batchSize,
-      maxBatches: args.maxBatches,
-    }) as Promise<SyncGitHubBackupsResult>
-  },
-})
-
-function clampInt(value: number, min: number, max: number) {
-  return Math.max(min, Math.min(max, Math.floor(value)))
-}

package/convex/githubBackupsNode.ts

@@ -1,183 +0,0 @@
-'use node'
-
-import { v } from 'convex/values'
-import { internal } from './_generated/api'
-import type { Doc } from './_generated/dataModel'
-import type { ActionCtx } from './_generated/server'
-import { internalAction } from './_generated/server'
-import {
-  backupSkillToGitHub,
-  fetchGitHubSkillMeta,
-  getGitHubBackupContext,
-  isGitHubBackupConfigured,
-} from './lib/githubBackup'
-
-const DEFAULT_BATCH_SIZE = 50
-const MAX_BATCH_SIZE = 200
-const DEFAULT_MAX_BATCHES = 5
-const MAX_MAX_BATCHES = 200
-
-type BackupPageItem =
-  | {
-      kind: 'ok'
-      slug: string
-      version: string
-      displayName: string
-      ownerHandle: string
-      files: Doc<'skillVersions'>['files']
-      publishedAt: number
-    }
-  | { kind: 'missingLatestVersion' }
-  | { kind: 'missingVersionDoc' }
-  | { kind: 'missingOwner' }
-
-export type GitHubBackupSyncStats = {
-  skillsScanned: number
-  skillsSkipped: number
-  skillsBackedUp: number
-  skillsMissingVersion: number
-  skillsMissingOwner: number
-  errors: number
-}
-
-export type SyncGitHubBackupsInternalArgs = {
-  dryRun?: boolean
-  batchSize?: number
-  maxBatches?: number
-}
-
-export type SyncGitHubBackupsInternalResult = {
-  stats: GitHubBackupSyncStats
-  cursor: string | null
-  isDone: boolean
-}
-
-export const backupSkillForPublishInternal = internalAction({
-  args: {
-    slug: v.string(),
-    version: v.string(),
-    displayName: v.string(),
-    ownerHandle: v.string(),
-    files: v.array(
-      v.object({
-        path: v.string(),
-        size: v.number(),
-        storageId: v.id('_storage'),
-        sha256: v.string(),
-        contentType: v.optional(v.string()),
-      }),
-    ),
-    publishedAt: v.number(),
-  },
-  handler: async (ctx, args) => {
-    if (!isGitHubBackupConfigured()) {
-      return { skipped: true as const }
-    }
-    await backupSkillToGitHub(ctx, args)
-    return { skipped: false as const }
-  },
-})
-
-export async function syncGitHubBackupsInternalHandler(
-  ctx: ActionCtx,
-  args: SyncGitHubBackupsInternalArgs,
-): Promise<SyncGitHubBackupsInternalResult> {
-  const dryRun = Boolean(args.dryRun)
-  const stats: GitHubBackupSyncStats = {
-    skillsScanned: 0,
-    skillsSkipped: 0,
-    skillsBackedUp: 0,
-    skillsMissingVersion: 0,
-    skillsMissingOwner: 0,
-    errors: 0,
-  }
-
-  if (!isGitHubBackupConfigured()) {
-    return { stats, cursor: null, isDone: true }
-  }
-
-  const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
-  const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES)
-  const context = await getGitHubBackupContext()
-
-  const state = dryRun
-    ? { cursor: null as string | null }
-    : ((await ctx.runQuery(internal.githubBackups.getGitHubBackupSyncStateInternal, {})) as {
-        cursor: string | null
-      })
-
-  let cursor: string | null = state.cursor
-  let isDone = false
-
-  for (let batch = 0; batch < maxBatches; batch++) {
-    const page = (await ctx.runQuery(internal.githubBackups.getGitHubBackupPageInternal, {
-      cursor: cursor ?? undefined,
-      batchSize,
-    })) as { items: BackupPageItem[]; cursor: string | null; isDone: boolean }
-
-    cursor = page.cursor
-    isDone = page.isDone
-
-    for (const item of page.items) {
-      if (item.kind !== 'ok') {
-        if (item.kind === 'missingLatestVersion' || item.kind === 'missingVersionDoc') {
-          stats.skillsMissingVersion += 1
-        } else if (item.kind === 'missingOwner') {
-          stats.skillsMissingOwner += 1
-        }
-        continue
-      }
-
-      stats.skillsScanned += 1
-      try {
-        const meta = await fetchGitHubSkillMeta(context, item.ownerHandle, item.slug)
-        if (meta?.latest?.version === item.version) {
-          stats.skillsSkipped += 1
-          continue
-        }
-
-        if (!dryRun) {
-          await backupSkillToGitHub(
-            ctx,
-            {
-              slug: item.slug,
-              version: item.version,
-              displayName: item.displayName,
-              ownerHandle: item.ownerHandle,
-              files: item.files,
-              publishedAt: item.publishedAt,
-            },
-            context,
-          )
-          stats.skillsBackedUp += 1
-        }
-      } catch (error) {
-        console.error('GitHub backup sync failed', error)
-        stats.errors += 1
-      }
-    }
-
-    if (!dryRun) {
-      await ctx.runMutation(internal.githubBackups.setGitHubBackupSyncStateInternal, {
-        cursor: isDone ? undefined : (cursor ?? undefined),
-      })
-    }
-
-    if (isDone) break
-  }
-
-  return { stats, cursor, isDone }
-}
-
-export const syncGitHubBackupsInternal = internalAction({
-  args: {
-    dryRun: v.optional(v.boolean()),
-    batchSize: v.optional(v.number()),
-    maxBatches: v.optional(v.number()),
-  },
-  handler: syncGitHubBackupsInternalHandler,
-})
-
-function clampInt(value: number, min: number, max: number) {
-  return Math.max(min, Math.min(max, Math.floor(value)))
-}

package/convex/githubImport.ts

@@ -1,317 +0,0 @@
-import { ConvexError, v } from 'convex/values'
-import { unzipSync } from 'fflate'
-import semver from 'semver'
-import { api, internal } from './_generated/api'
-import type { Id } from './_generated/dataModel'
-import type { ActionCtx } from './_generated/server'
-import { action } from './_generated/server'
-import { requireUserFromAction } from './lib/access'
-import {
-  buildGitHubImportFileList,
-  computeDefaultSelectedPaths,
-  detectGitHubImportCandidates,
-  fetchGitHubZipBytes,
-  listTextFilesUnderCandidate,
-  normalizeRepoPath,
-  parseGitHubImportUrl,
-  resolveGitHubCommit,
-  stripGitHubZipRoot,
-  suggestDisplayName,
-  suggestVersion,
-} from './lib/githubImport'
-import { publishVersionForUser } from './lib/skillPublish'
-import { sanitizePath } from './lib/skills'
-
-const MAX_SELECTED_BYTES = 50 * 1024 * 1024
-const MAX_UNZIPPED_BYTES = 80 * 1024 * 1024
-const MAX_FILE_COUNT = 7_500
-const MAX_SINGLE_FILE_BYTES = 10 * 1024 * 1024
-
-export const previewGitHubImport = action({
-  args: { url: v.string() },
-  handler: async (ctx, args) => {
-    await requireUserFromAction(ctx)
-
-    const parsed = parseGitHubImportUrl(args.url)
-    const resolved = await resolveGitHubCommit(parsed, fetch)
-    const zipBytes = await fetchGitHubZipBytes(resolved, fetch)
-    const entries = unzipToEntries(zipBytes)
-    const stripped = stripGitHubZipRoot(entries)
-    const candidates = detectGitHubImportCandidates(stripped).filter((candidate) =>
-      isCandidateUnderResolvedPath(candidate.path, resolved.path),
-    )
-    if (candidates.length === 0) throw new ConvexError('No SKILL.md found in this repo')
-
-    return {
-      resolved,
-      candidates: candidates.map((candidate) => ({
-        path: candidate.path,
-        readmePath: candidate.readmePath,
-        name: candidate.name ?? null,
-        description: candidate.description ?? null,
-      })),
-    }
-  },
-})
-
-export const previewGitHubImportCandidate = action({
-  args: { url: v.string(), candidatePath: v.string() },
-  handler: async (ctx, args) => {
-    const { userId } = await requireUserFromAction(ctx)
-
-    const parsed = parseGitHubImportUrl(args.url)
-    const resolved = await resolveGitHubCommit(parsed, fetch)
-    const zipBytes = await fetchGitHubZipBytes(resolved, fetch)
-    const entries = unzipToEntries(zipBytes)
-    const stripped = stripGitHubZipRoot(entries)
-
-    const normalizedCandidatePath = normalizeRepoPath(args.candidatePath)
-    if (!isCandidateUnderResolvedPath(normalizedCandidatePath, resolved.path)) {
-      throw new ConvexError('Candidate path is outside the requested import scope')
-    }
-
-    const candidates = detectGitHubImportCandidates(stripped).filter((candidate) =>
-      isCandidateUnderResolvedPath(candidate.path, resolved.path),
-    )
-
-    const candidate = candidates.find((item) => item.path === normalizedCandidatePath)
-    if (!candidate) throw new ConvexError('Candidate not found')
-
-    const files = listTextFilesUnderCandidate(stripped, candidate.path)
-    const defaultSelectedPaths = computeDefaultSelectedPaths({ candidate, files })
-    const fileList = buildGitHubImportFileList({
-      candidate,
-      files,
-      defaultSelectedPaths,
-    })
-
-    const baseForNaming = candidate.path ? (candidate.path.split('/').at(-1) ?? '') : resolved.repo
-    const suggestedDisplayName = suggestDisplayName(candidate, baseForNaming)
-
-    const rawSlugBase = sanitizeSlug(candidate.path ? baseForNaming : resolved.repo)
-    const suggestedSlug = await suggestAvailableSlug(ctx, userId, rawSlugBase)
-
-    const existing = await ctx.runQuery(api.skills.getBySlug, { slug: suggestedSlug })
-    const existingLatest =
-      existing?.skill && existing.skill.ownerUserId === userId
-        ? (existing.latestVersion?.version ?? null)
-        : null
-    const suggestedVersion = suggestVersion(existingLatest)
-
-    return {
-      resolved,
-      candidate: {
-        path: candidate.path,
-        readmePath: candidate.readmePath,
-        name: candidate.name ?? null,
-        description: candidate.description ?? null,
-      },
-      defaults: {
-        selectedPaths: defaultSelectedPaths,
-        slug: suggestedSlug,
-        displayName: suggestedDisplayName,
-        version: suggestedVersion,
-        tags: ['latest'],
-      },
-      files: fileList,
-    }
-  },
-})
-
-export const importGitHubSkill = action({
-  args: {
-    url: v.string(),
-    commit: v.string(),
-    candidatePath: v.string(),
-    selectedPaths: v.array(v.string()),
-    slug: v.optional(v.string()),
-    displayName: v.optional(v.string()),
-    version: v.optional(v.string()),
-    tags: v.optional(v.array(v.string())),
-  },
-  handler: async (ctx, args) => {
-    const { userId } = await requireUserFromAction(ctx)
-
-    const parsed = parseGitHubImportUrl(args.url)
-    const resolved = await resolveGitHubCommit(parsed, fetch)
-    if (!/^[a-f0-9]{40}$/i.test(args.commit)) throw new ConvexError('Invalid commit')
-    if (args.commit.toLowerCase() !== resolved.commit.toLowerCase()) {
-      throw new ConvexError('Import is out of date. Re-run preview.')
-    }
-
-    const normalizedCandidatePath = normalizeRepoPath(args.candidatePath)
-    if (!isCandidateUnderResolvedPath(normalizedCandidatePath, resolved.path)) {
-      throw new ConvexError('Candidate path is outside the requested import scope')
-    }
-
-    const zipBytes = await fetchGitHubZipBytes(resolved, fetch)
-    const entries = stripGitHubZipRoot(unzipToEntries(zipBytes))
-
-    const candidates = detectGitHubImportCandidates(entries).filter((candidate) =>
-      isCandidateUnderResolvedPath(candidate.path, resolved.path),
-    )
-    const candidate = candidates.find((item) => item.path === normalizedCandidatePath)
-    if (!candidate) throw new ConvexError('Candidate not found')
-
-    const filesUnderCandidate = listTextFilesUnderCandidate(entries, candidate.path)
-    const byPath = new Map(filesUnderCandidate.map((file) => [file.path, file.bytes]))
-
-    const selected = Array.from(
-      new Set(args.selectedPaths.map((path) => normalizeRepoPath(path)).filter(Boolean)),
-    )
-    if (selected.length === 0) throw new ConvexError('No files selected')
-
-    const candidateRoot = candidate.path ? `${candidate.path}/` : ''
-    const normalizedReadmePath = normalizeRepoPath(candidate.readmePath)
-    if (!selected.includes(normalizedReadmePath)) {
-      throw new ConvexError('SKILL.md must be selected')
-    }
-
-    let totalBytes = 0
-    const storedFiles: Array<{
-      path: string
-      size: number
-      storageId: Id<'_storage'>
-      sha256: string
-      contentType?: string
-    }> = []
-
-    for (const path of selected.sort()) {
-      if (candidateRoot && !path.startsWith(candidateRoot)) {
-        throw new ConvexError('Selected file is outside the chosen skill folder')
-      }
-
-      const bytes = byPath.get(path)
-      if (!bytes) continue
-      totalBytes += bytes.byteLength
-      if (totalBytes > MAX_SELECTED_BYTES) throw new ConvexError('Selected files exceed 50MB limit')
-
-      const relPath = candidateRoot ? path.slice(candidateRoot.length) : path
-      const sanitized = sanitizePath(relPath)
-      if (!sanitized) throw new ConvexError('Invalid file paths')
-
-      const sha256 = await sha256Hex(bytes)
-      const safeBytes = new Uint8Array(bytes)
-      const storageId = await ctx.storage.store(new Blob([safeBytes], { type: 'text/plain' }))
-      storedFiles.push({
-        path: sanitized,
-        size: bytes.byteLength,
-        storageId,
-        sha256,
-        contentType: 'text/plain',
-      })
-    }
-
-    if (storedFiles.length === 0) throw new ConvexError('No files selected')
-
-    const slugBase = (args.slug ?? '').trim().toLowerCase()
-    const displayName = (args.displayName ?? '').trim()
-    const tags = (args.tags ?? ['latest']).map((tag) => tag.trim()).filter(Boolean)
-    const version = (args.version ?? '').trim()
-
-    if (!slugBase) throw new ConvexError('Slug required')
-    if (!displayName) throw new ConvexError('Display name required')
-    if (!version || !semver.valid(version)) throw new ConvexError('Version must be valid semver')
-
-    const result = await publishVersionForUser(ctx, userId, {
-      slug: slugBase,
-      displayName,
-      version,
-      changelog: '',
-      tags,
-      files: storedFiles,
-      source: {
-        kind: 'github',
-        url: resolved.originalUrl,
-        repo: `${resolved.owner}/${resolved.repo}`,
-        ref: resolved.ref,
-        commit: resolved.commit,
-        path: candidate.path,
-        importedAt: Date.now(),
-      },
-    })
-
-    return { ok: true, slug: slugBase, version, ...result }
-  },
-})
-
-function unzipToEntries(zipBytes: Uint8Array) {
-  const entries = unzipSync(zipBytes)
-  const out: Record<string, Uint8Array> = {}
-  const rawPaths = Object.keys(entries)
-  if (rawPaths.length > MAX_FILE_COUNT) throw new ConvexError('Repo archive has too many files')
-  let totalBytes = 0
-  for (const [rawPath, bytes] of Object.entries(entries)) {
-    const normalizedPath = normalizeZipPath(rawPath)
-    if (!normalizedPath) continue
-    if (isJunkPath(normalizedPath)) continue
-    if (!bytes) continue
-    if (bytes.byteLength > MAX_SINGLE_FILE_BYTES) continue
-    totalBytes += bytes.byteLength
-    if (totalBytes > MAX_UNZIPPED_BYTES) throw new ConvexError('Repo archive is too large')
-    out[normalizedPath] = bytes
-  }
-  return out
-}
-
-function isCandidateUnderResolvedPath(candidatePath: string, resolvedPath: string) {
-  const root = normalizeRepoPath(resolvedPath)
-  if (!root) return true
-  if (!candidatePath) return false
-  if (candidatePath === root) return true
-  return candidatePath.startsWith(`${root}/`)
-}
-
-function sanitizeSlug(value: string) {
-  return value
-    .trim()
-    .toLowerCase()
-    .replace(/[^a-z0-9-]+/g, '-')
-    .replace(/^-+/, '')
-    .replace(/-+$/, '')
-    .replace(/--+/g, '-')
-}
-
-async function suggestAvailableSlug(ctx: ActionCtx, userId: Id<'users'>, base: string) {
-  const cleaned = sanitizeSlug(base)
-  if (!cleaned) throw new ConvexError('Could not derive slug')
-  for (let i = 0; i < 50; i += 1) {
-    const candidate = i === 0 ? cleaned : `${cleaned}-${i + 1}`
-    const existing = await ctx.runQuery(internal.skills.getSkillBySlugInternal, { slug: candidate })
-    if (!existing) return candidate
-    if (existing.ownerUserId === userId) return candidate
-  }
-  throw new ConvexError('Could not find an available slug')
-}
-
-async function sha256Hex(bytes: Uint8Array) {
-  const normalized = new Uint8Array(bytes)
-  const digest = await crypto.subtle.digest('SHA-256', normalized.buffer)
-  return toHex(new Uint8Array(digest))
-}
-
-function toHex(bytes: Uint8Array) {
-  let out = ''
-  for (const byte of bytes) out += byte.toString(16).padStart(2, '0')
-  return out
-}
-
-function normalizeZipPath(path: string) {
-  const normalized = path
-    .replaceAll('\u0000', '')
-    .replaceAll('\\', '/')
-    .trim()
-    .replace(/^\.\/+/, '')
-    .replace(/^\/+/, '')
-  if (!normalized) return ''
-  if (normalized.includes('..')) return ''
-  return normalized
-}
-
-function isJunkPath(path: string) {
-  const normalized = path.toLowerCase()
-  if (normalized.startsWith('__macosx/')) return true
-  if (normalized.endsWith('/.ds_store')) return true
-  if (normalized === '.ds_store') return true
-  return false
-}