pilothub 0.0.1

package/convex/githubSoulBackups.ts

@@ -0,0 +1,170 @@
+import { v } from 'convex/values'
+import { internal } from './_generated/api'
+import type { Doc, Id } from './_generated/dataModel'
+import { action, internalMutation, internalQuery } from './_generated/server'
+import { assertRole, requireUserFromAction } from './lib/access'
+
+const DEFAULT_BATCH_SIZE = 50
+const MAX_BATCH_SIZE = 200
+const SYNC_STATE_KEY = 'souls'
+
+type BackupPageItem =
+  | {
+      kind: 'ok'
+      soulId: Id<'souls'>
+      versionId: Id<'soulVersions'>
+      slug: string
+      displayName: string
+      version: string
+      ownerHandle: string
+      files: Doc<'soulVersions'>['files']
+      publishedAt: number
+    }
+  | { kind: 'missingLatestVersion'; soulId: Id<'souls'> }
+  | { kind: 'missingVersionDoc'; soulId: Id<'souls'>; versionId: Id<'soulVersions'> }
+  | { kind: 'missingOwner'; soulId: Id<'souls'>; ownerUserId: Id<'users'> }
+
+type BackupPageResult = {
+  items: BackupPageItem[]
+  cursor: string | null
+  isDone: boolean
+}
+
+type BackupSyncState = {
+  cursor: string | null
+}
+
+export type SyncGitHubSoulBackupsResult = {
+  stats: {
+    soulsScanned: number
+    soulsSkipped: number
+    soulsBackedUp: number
+    soulsMissingVersion: number
+    soulsMissingOwner: number
+    errors: number
+  }
+  cursor: string | null
+  isDone: boolean
+}
+
+export const getGitHubSoulBackupPageInternal = internalQuery({
+  args: {
+    cursor: v.optional(v.string()),
+    batchSize: v.optional(v.number()),
+  },
+  handler: async (ctx, args): Promise<BackupPageResult> => {
+    const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
+    const { page, isDone, continueCursor } = await ctx.db
+      .query('souls')
+      .order('asc')
+      .paginate({ cursor: args.cursor ?? null, numItems: batchSize })
+
+    const items: BackupPageItem[] = []
+    for (const soul of page) {
+      if (soul.softDeletedAt) continue
+      if (!soul.latestVersionId) {
+        items.push({ kind: 'missingLatestVersion', soulId: soul._id })
+        continue
+      }
+
+      const version = await ctx.db.get(soul.latestVersionId)
+      if (!version) {
+        items.push({
+          kind: 'missingVersionDoc',
+          soulId: soul._id,
+          versionId: soul.latestVersionId,
+        })
+        continue
+      }
+
+      const owner = await ctx.db.get(soul.ownerUserId)
+      if (!owner || owner.deletedAt) {
+        items.push({ kind: 'missingOwner', soulId: soul._id, ownerUserId: soul.ownerUserId })
+        continue
+      }
+
+      items.push({
+        kind: 'ok',
+        soulId: soul._id,
+        versionId: version._id,
+        slug: soul.slug,
+        displayName: soul.displayName,
+        version: version.version,
+        ownerHandle: owner.handle ?? owner._id,
+        files: version.files,
+        publishedAt: version.createdAt,
+      })
+    }
+
+    return { items, cursor: continueCursor, isDone }
+  },
+})
+
+export const getGitHubSoulBackupSyncStateInternal = internalQuery({
+  args: {},
+  handler: async (ctx): Promise<BackupSyncState> => {
+    const state = await ctx.db
+      .query('githubBackupSyncState')
+      .withIndex('by_key', (q) => q.eq('key', SYNC_STATE_KEY))
+      .unique()
+    return { cursor: state?.cursor ?? null }
+  },
+})
+
+export const setGitHubSoulBackupSyncStateInternal = internalMutation({
+  args: {
+    cursor: v.optional(v.string()),
+  },
+  handler: async (ctx, args) => {
+    const now = Date.now()
+    const state = await ctx.db
+      .query('githubBackupSyncState')
+      .withIndex('by_key', (q) => q.eq('key', SYNC_STATE_KEY))
+      .unique()
+
+    if (!state) {
+      await ctx.db.insert('githubBackupSyncState', {
+        key: SYNC_STATE_KEY,
+        cursor: args.cursor,
+        updatedAt: now,
+      })
+      return { ok: true as const }
+    }
+
+    await ctx.db.patch(state._id, {
+      cursor: args.cursor,
+      updatedAt: now,
+    })
+
+    return { ok: true as const }
+  },
+})
+
+export const syncGitHubSoulBackups: ReturnType<typeof action> = action({
+  args: {
+    dryRun: v.optional(v.boolean()),
+    batchSize: v.optional(v.number()),
+    maxBatches: v.optional(v.number()),
+    resetCursor: v.optional(v.boolean()),
+  },
+  handler: async (ctx, args): Promise<SyncGitHubSoulBackupsResult> => {
+    const { user } = await requireUserFromAction(ctx)
+    assertRole(user, ['admin'])
+
+    if (args.resetCursor && !args.dryRun) {
+      await ctx.runMutation(internal.githubSoulBackups.setGitHubSoulBackupSyncStateInternal, {
+        cursor: undefined,
+      })
+    }
+
+    return ctx.runAction(internal.githubSoulBackupsNode.syncGitHubSoulBackupsInternal, {
+      dryRun: args.dryRun,
+      batchSize: args.batchSize,
+      maxBatches: args.maxBatches,
+    }) as Promise<SyncGitHubSoulBackupsResult>
+  },
+})
+
+function clampInt(value: number, min: number, max: number) {
+  return Math.max(min, Math.min(max, Math.floor(value)))
+}

package/convex/githubSoulBackupsNode.ts

@@ -0,0 +1,186 @@
+'use node'
+
+import { v } from 'convex/values'
+import { internal } from './_generated/api'
+import type { Doc } from './_generated/dataModel'
+import type { ActionCtx } from './_generated/server'
+import { internalAction } from './_generated/server'
+import {
+  backupSoulToGitHub,
+  fetchGitHubSoulMeta,
+  getGitHubSoulBackupContext,
+  isGitHubSoulBackupConfigured,
+} from './lib/githubSoulBackup'
+
+const DEFAULT_BATCH_SIZE = 50
+const MAX_BATCH_SIZE = 200
+const DEFAULT_MAX_BATCHES = 5
+const MAX_MAX_BATCHES = 200
+
+type BackupPageItem =
+  | {
+      kind: 'ok'
+      slug: string
+      version: string
+      displayName: string
+      ownerHandle: string
+      files: Doc<'soulVersions'>['files']
+      publishedAt: number
+    }
+  | { kind: 'missingLatestVersion' }
+  | { kind: 'missingVersionDoc' }
+  | { kind: 'missingOwner' }
+
+export type GitHubSoulBackupSyncStats = {
+  soulsScanned: number
+  soulsSkipped: number
+  soulsBackedUp: number
+  soulsMissingVersion: number
+  soulsMissingOwner: number
+  errors: number
+}
+
+export type SyncGitHubSoulBackupsInternalArgs = {
+  dryRun?: boolean
+  batchSize?: number
+  maxBatches?: number
+}
+
+export type SyncGitHubSoulBackupsInternalResult = {
+  stats: GitHubSoulBackupSyncStats
+  cursor: string | null
+  isDone: boolean
+}
+
+export const backupSoulForPublishInternal = internalAction({
+  args: {
+    slug: v.string(),
+    version: v.string(),
+    displayName: v.string(),
+    ownerHandle: v.string(),
+    files: v.array(
+      v.object({
+        path: v.string(),
+        size: v.number(),
+        storageId: v.id('_storage'),
+        sha256: v.string(),
+        contentType: v.optional(v.string()),
+      }),
+    ),
+    publishedAt: v.number(),
+  },
+  handler: async (ctx, args) => {
+    if (!isGitHubSoulBackupConfigured()) {
+      return { skipped: true as const }
+    }
+    await backupSoulToGitHub(ctx, args)
+    return { skipped: false as const }
+  },
+})
+
+export async function syncGitHubSoulBackupsInternalHandler(
+  ctx: ActionCtx,
+  args: SyncGitHubSoulBackupsInternalArgs,
+): Promise<SyncGitHubSoulBackupsInternalResult> {
+  const dryRun = Boolean(args.dryRun)
+  const stats: GitHubSoulBackupSyncStats = {
+    soulsScanned: 0,
+    soulsSkipped: 0,
+    soulsBackedUp: 0,
+    soulsMissingVersion: 0,
+    soulsMissingOwner: 0,
+    errors: 0,
+  }
+
+  if (!isGitHubSoulBackupConfigured()) {
+    return { stats, cursor: null, isDone: true }
+  }
+
+  const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
+  const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES)
+  const context = await getGitHubSoulBackupContext()
+
+  const state = dryRun
+    ? { cursor: null as string | null }
+    : ((await ctx.runQuery(
+        internal.githubSoulBackups.getGitHubSoulBackupSyncStateInternal,
+        {},
+      )) as {
+        cursor: string | null
+      })
+
+  let cursor: string | null = state.cursor
+  let isDone = false
+
+  for (let batch = 0; batch < maxBatches; batch++) {
+    const page = (await ctx.runQuery(internal.githubSoulBackups.getGitHubSoulBackupPageInternal, {
+      cursor: cursor ?? undefined,
+      batchSize,
+    })) as { items: BackupPageItem[]; cursor: string | null; isDone: boolean }
+
+    cursor = page.cursor
+    isDone = page.isDone
+
+    for (const item of page.items) {
+      if (item.kind !== 'ok') {
+        if (item.kind === 'missingLatestVersion' || item.kind === 'missingVersionDoc') {
+          stats.soulsMissingVersion += 1
+        } else if (item.kind === 'missingOwner') {
+          stats.soulsMissingOwner += 1
+        }
+        continue
+      }
+
+      stats.soulsScanned += 1
+      try {
+        const meta = await fetchGitHubSoulMeta(context, item.ownerHandle, item.slug)
+        if (meta?.latest?.version === item.version) {
+          stats.soulsSkipped += 1
+          continue
+        }
+
+        if (!dryRun) {
+          await backupSoulToGitHub(
+            ctx,
+            {
+              slug: item.slug,
+              version: item.version,
+              displayName: item.displayName,
+              ownerHandle: item.ownerHandle,
+              files: item.files,
+              publishedAt: item.publishedAt,
+            },
+            context,
+          )
+          stats.soulsBackedUp += 1
+        }
+      } catch (error) {
+        console.error('GitHub soul backup sync failed', error)
+        stats.errors += 1
+      }
+    }
+
+    if (!dryRun) {
+      await ctx.runMutation(internal.githubSoulBackups.setGitHubSoulBackupSyncStateInternal, {
+        cursor: isDone ? undefined : (cursor ?? undefined),
+      })
+    }
+
+    if (isDone) break
+  }
+
+  return { stats, cursor, isDone }
+}
+
+export const syncGitHubSoulBackupsInternal = internalAction({
+  args: {
+    dryRun: v.optional(v.boolean()),
+    batchSize: v.optional(v.number()),
+    maxBatches: v.optional(v.number()),
+  },
+  handler: syncGitHubSoulBackupsInternalHandler,
+})
+
+function clampInt(value: number, min: number, max: number) {
+  return Math.max(min, Math.min(max, Math.floor(value)))
+}

package/convex/http.ts

@@ -0,0 +1,194 @@
+import { httpRouter } from 'convex/server'
+import { ApiRoutes, LegacyApiRoutes } from 'pilothub-schema'
+import { auth } from './auth'
+import { downloadZip } from './downloads'
+import {
+  cliPublishHttp,
+  cliSkillDeleteHttp,
+  cliSkillUndeleteHttp,
+  cliTelemetrySyncHttp,
+  cliUploadUrlHttp,
+  cliWhoamiHttp,
+  getSkillHttp,
+  resolveSkillVersionHttp,
+  searchSkillsHttp,
+} from './httpApi'
+import {
+  listSkillsV1Http,
+  listSoulsV1Http,
+  publishSkillV1Http,
+  publishSoulV1Http,
+  resolveSkillVersionV1Http,
+  searchSkillsV1Http,
+  skillsDeleteRouterV1Http,
+  skillsGetRouterV1Http,
+  skillsPostRouterV1Http,
+  soulsDeleteRouterV1Http,
+  soulsGetRouterV1Http,
+  soulsPostRouterV1Http,
+  starsDeleteRouterV1Http,
+  starsPostRouterV1Http,
+  whoamiV1Http,
+} from './httpApiV1'
+
+const http = httpRouter()
+
+auth.addHttpRoutes(http)
+
+http.route({
+  path: ApiRoutes.download,
+  method: 'GET',
+  handler: downloadZip,
+})
+
+http.route({
+  path: ApiRoutes.search,
+  method: 'GET',
+  handler: searchSkillsV1Http,
+})
+
+http.route({
+  path: ApiRoutes.resolve,
+  method: 'GET',
+  handler: resolveSkillVersionV1Http,
+})
+
+http.route({
+  path: ApiRoutes.skills,
+  method: 'GET',
+  handler: listSkillsV1Http,
+})
+
+http.route({
+  pathPrefix: `${ApiRoutes.skills}/`,
+  method: 'GET',
+  handler: skillsGetRouterV1Http,
+})
+
+http.route({
+  path: ApiRoutes.skills,
+  method: 'POST',
+  handler: publishSkillV1Http,
+})
+
+http.route({
+  pathPrefix: `${ApiRoutes.skills}/`,
+  method: 'POST',
+  handler: skillsPostRouterV1Http,
+})
+
+http.route({
+  pathPrefix: `${ApiRoutes.skills}/`,
+  method: 'DELETE',
+  handler: skillsDeleteRouterV1Http,
+})
+
+http.route({
+  pathPrefix: `${ApiRoutes.stars}/`,
+  method: 'POST',
+  handler: starsPostRouterV1Http,
+})
+
+http.route({
+  pathPrefix: `${ApiRoutes.stars}/`,
+  method: 'DELETE',
+  handler: starsDeleteRouterV1Http,
+})
+
+http.route({
+  path: ApiRoutes.whoami,
+  method: 'GET',
+  handler: whoamiV1Http,
+})
+
+http.route({
+  path: ApiRoutes.souls,
+  method: 'GET',
+  handler: listSoulsV1Http,
+})
+
+http.route({
+  pathPrefix: `${ApiRoutes.souls}/`,
+  method: 'GET',
+  handler: soulsGetRouterV1Http,
+})
+
+http.route({
+  path: ApiRoutes.souls,
+  method: 'POST',
+  handler: publishSoulV1Http,
+})
+
+http.route({
+  pathPrefix: `${ApiRoutes.souls}/`,
+  method: 'POST',
+  handler: soulsPostRouterV1Http,
+})
+
+http.route({
+  pathPrefix: `${ApiRoutes.souls}/`,
+  method: 'DELETE',
+  handler: soulsDeleteRouterV1Http,
+})
+
+// TODO: remove legacy /api routes after deprecation window.
+http.route({
+  path: LegacyApiRoutes.download,
+  method: 'GET',
+  handler: downloadZip,
+})
+http.route({
+  path: LegacyApiRoutes.search,
+  method: 'GET',
+  handler: searchSkillsHttp,
+})
+
+http.route({
+  path: LegacyApiRoutes.skill,
+  method: 'GET',
+  handler: getSkillHttp,
+})
+
+http.route({
+  path: LegacyApiRoutes.skillResolve,
+  method: 'GET',
+  handler: resolveSkillVersionHttp,
+})
+
+http.route({
+  path: LegacyApiRoutes.cliWhoami,
+  method: 'GET',
+  handler: cliWhoamiHttp,
+})
+
+http.route({
+  path: LegacyApiRoutes.cliUploadUrl,
+  method: 'POST',
+  handler: cliUploadUrlHttp,
+})
+
+http.route({
+  path: LegacyApiRoutes.cliPublish,
+  method: 'POST',
+  handler: cliPublishHttp,
+})
+
+http.route({
+  path: LegacyApiRoutes.cliTelemetrySync,
+  method: 'POST',
+  handler: cliTelemetrySyncHttp,
+})
+
+http.route({
+  path: LegacyApiRoutes.cliSkillDelete,
+  method: 'POST',
+  handler: cliSkillDeleteHttp,
+})
+
+http.route({
+  path: LegacyApiRoutes.cliSkillUndelete,
+  method: 'POST',
+  handler: cliSkillUndeleteHttp,
+})
+
+export default http