pilothub 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (272) hide show
  1. package/.env.local.example +19 -0
  2. package/.github/workflows/ci.yml +40 -0
  3. package/.oxlintrc.json +3 -0
  4. package/AGENTS.md +45 -0
  5. package/CHANGELOG.md +138 -0
  6. package/DEPRECATIONS.md +7 -0
  7. package/LICENSE +21 -0
  8. package/README.md +150 -0
  9. package/biome.json +41 -0
  10. package/convex/_generated/api.d.ts +153 -0
  11. package/convex/_generated/api.js +23 -0
  12. package/convex/_generated/dataModel.d.ts +60 -0
  13. package/convex/_generated/server.d.ts +143 -0
  14. package/convex/_generated/server.js +93 -0
  15. package/convex/auth.config.ts +8 -0
  16. package/convex/auth.ts +19 -0
  17. package/convex/comments.ts +88 -0
  18. package/convex/crons.ts +34 -0
  19. package/convex/devSeed.ts +459 -0
  20. package/convex/devSeedExtra.ts +541 -0
  21. package/convex/downloads.ts +78 -0
  22. package/convex/githubBackups.ts +170 -0
  23. package/convex/githubBackupsNode.ts +183 -0
  24. package/convex/githubImport.ts +317 -0
  25. package/convex/githubSoulBackups.ts +170 -0
  26. package/convex/githubSoulBackupsNode.ts +186 -0
  27. package/convex/http.ts +194 -0
  28. package/convex/httpApi.handlers.test.ts +488 -0
  29. package/convex/httpApi.test.ts +70 -0
  30. package/convex/httpApi.ts +305 -0
  31. package/convex/httpApiV1.handlers.test.ts +584 -0
  32. package/convex/httpApiV1.ts +1172 -0
  33. package/convex/leaderboards.ts +39 -0
  34. package/convex/lib/access.ts +36 -0
  35. package/convex/lib/apiTokenAuth.ts +36 -0
  36. package/convex/lib/badges.ts +50 -0
  37. package/convex/lib/changelog.test.ts +34 -0
  38. package/convex/lib/changelog.ts +278 -0
  39. package/convex/lib/embeddings.ts +38 -0
  40. package/convex/lib/githubBackup.ts +443 -0
  41. package/convex/lib/githubImport.test.ts +247 -0
  42. package/convex/lib/githubImport.ts +425 -0
  43. package/convex/lib/githubSoulBackup.ts +443 -0
  44. package/convex/lib/leaderboards.ts +103 -0
  45. package/convex/lib/moderation.ts +42 -0
  46. package/convex/lib/public.ts +89 -0
  47. package/convex/lib/searchText.test.ts +46 -0
  48. package/convex/lib/searchText.ts +27 -0
  49. package/convex/lib/skillBackfill.test.ts +34 -0
  50. package/convex/lib/skillBackfill.ts +67 -0
  51. package/convex/lib/skillPublish.test.ts +28 -0
  52. package/convex/lib/skillPublish.ts +284 -0
  53. package/convex/lib/skillStats.ts +80 -0
  54. package/convex/lib/skills.test.ts +197 -0
  55. package/convex/lib/skills.ts +273 -0
  56. package/convex/lib/soulChangelog.ts +273 -0
  57. package/convex/lib/soulPublish.ts +236 -0
  58. package/convex/lib/tokens.test.ts +33 -0
  59. package/convex/lib/tokens.ts +51 -0
  60. package/convex/lib/webhooks.test.ts +91 -0
  61. package/convex/lib/webhooks.ts +112 -0
  62. package/convex/maintenance.test.ts +270 -0
  63. package/convex/maintenance.ts +840 -0
  64. package/convex/rateLimits.ts +50 -0
  65. package/convex/schema.ts +472 -0
  66. package/convex/search.test.ts +12 -0
  67. package/convex/search.ts +254 -0
  68. package/convex/seed.test.ts +37 -0
  69. package/convex/seed.ts +254 -0
  70. package/convex/seedSouls.ts +111 -0
  71. package/convex/skillStatEvents.ts +568 -0
  72. package/convex/skills.ts +1606 -0
  73. package/convex/soulComments.ts +88 -0
  74. package/convex/soulDownloads.ts +14 -0
  75. package/convex/soulStars.ts +71 -0
  76. package/convex/souls.ts +570 -0
  77. package/convex/stars.ts +108 -0
  78. package/convex/statsMaintenance.ts +205 -0
  79. package/convex/telemetry.ts +434 -0
  80. package/convex/tokens.ts +88 -0
  81. package/convex/tsconfig.json +7 -0
  82. package/convex/uploads.ts +20 -0
  83. package/convex/users.ts +122 -0
  84. package/convex/webhooks.ts +50 -0
  85. package/convex.json +3 -0
  86. package/docs/README.md +32 -0
  87. package/docs/api.md +51 -0
  88. package/docs/architecture.md +61 -0
  89. package/docs/auth.md +54 -0
  90. package/docs/cli.md +117 -0
  91. package/docs/deploy.md +78 -0
  92. package/docs/diffing.md +84 -0
  93. package/docs/github-import.md +171 -0
  94. package/docs/http-api.md +187 -0
  95. package/docs/manual-testing.md +64 -0
  96. package/docs/mintlify.md +43 -0
  97. package/docs/quickstart.md +120 -0
  98. package/docs/skill-format.md +58 -0
  99. package/docs/soul-format.md +37 -0
  100. package/docs/spec.md +177 -0
  101. package/docs/telemetry.md +91 -0
  102. package/docs/troubleshooting.md +49 -0
  103. package/docs/webhook.md +51 -0
  104. package/e2e/menu-smoke.pw.test.ts +49 -0
  105. package/e2e/pilothub.e2e.test.ts +494 -0
  106. package/e2e/search-exact.pw.test.ts +97 -0
  107. package/package.json +84 -0
  108. package/packages/pilothub/LICENSE +22 -0
  109. package/packages/pilothub/README.md +57 -0
  110. package/packages/pilothub/bin/pilothub.js +2 -0
  111. package/packages/pilothub/package.json +41 -0
  112. package/packages/pilothub/src/browserAuth.test.ts +96 -0
  113. package/packages/pilothub/src/browserAuth.ts +174 -0
  114. package/packages/pilothub/src/cli/buildInfo.ts +94 -0
  115. package/packages/pilothub/src/cli/commands/auth.ts +97 -0
  116. package/packages/pilothub/src/cli/commands/delete.test.ts +73 -0
  117. package/packages/pilothub/src/cli/commands/delete.ts +83 -0
  118. package/packages/pilothub/src/cli/commands/publish.test.ts +122 -0
  119. package/packages/pilothub/src/cli/commands/publish.ts +108 -0
  120. package/packages/pilothub/src/cli/commands/skills.test.ts +191 -0
  121. package/packages/pilothub/src/cli/commands/skills.ts +380 -0
  122. package/packages/pilothub/src/cli/commands/star.ts +46 -0
  123. package/packages/pilothub/src/cli/commands/sync.test.ts +310 -0
  124. package/packages/pilothub/src/cli/commands/sync.ts +200 -0
  125. package/packages/pilothub/src/cli/commands/syncHelpers.test.ts +26 -0
  126. package/packages/pilothub/src/cli/commands/syncHelpers.ts +427 -0
  127. package/packages/pilothub/src/cli/commands/syncTypes.ts +27 -0
  128. package/packages/pilothub/src/cli/commands/unstar.ts +48 -0
  129. package/packages/pilothub/src/cli/helpStyle.ts +45 -0
  130. package/packages/pilothub/src/cli/pilotbotConfig.test.ts +159 -0
  131. package/packages/pilothub/src/cli/pilotbotConfig.ts +147 -0
  132. package/packages/pilothub/src/cli/registry.test.ts +63 -0
  133. package/packages/pilothub/src/cli/registry.ts +43 -0
  134. package/packages/pilothub/src/cli/scanSkills.test.ts +64 -0
  135. package/packages/pilothub/src/cli/scanSkills.ts +84 -0
  136. package/packages/pilothub/src/cli/slug.ts +16 -0
  137. package/packages/pilothub/src/cli/types.ts +12 -0
  138. package/packages/pilothub/src/cli/ui.ts +75 -0
  139. package/packages/pilothub/src/cli.ts +311 -0
  140. package/packages/pilothub/src/config.ts +36 -0
  141. package/packages/pilothub/src/discovery.test.ts +75 -0
  142. package/packages/pilothub/src/discovery.ts +19 -0
  143. package/packages/pilothub/src/http.test.ts +156 -0
  144. package/packages/pilothub/src/http.ts +301 -0
  145. package/packages/pilothub/src/schema/ark.ts +29 -0
  146. package/packages/pilothub/src/schema/index.ts +5 -0
  147. package/packages/pilothub/src/schema/routes.ts +22 -0
  148. package/packages/pilothub/src/schema/schemas.ts +260 -0
  149. package/packages/pilothub/src/schema/textFiles.test.ts +23 -0
  150. package/packages/pilothub/src/schema/textFiles.ts +66 -0
  151. package/packages/pilothub/src/skills.test.ts +191 -0
  152. package/packages/pilothub/src/skills.ts +172 -0
  153. package/packages/pilothub/src/types.ts +10 -0
  154. package/packages/pilothub/tsconfig.json +14 -0
  155. package/packages/schema/README.md +3 -0
  156. package/packages/schema/dist/ark.d.ts +4 -0
  157. package/packages/schema/dist/ark.js +26 -0
  158. package/packages/schema/dist/ark.js.map +1 -0
  159. package/packages/schema/dist/index.d.ts +5 -0
  160. package/packages/schema/dist/index.js +5 -0
  161. package/packages/schema/dist/index.js.map +1 -0
  162. package/packages/schema/dist/routes.d.ts +21 -0
  163. package/packages/schema/dist/routes.js +22 -0
  164. package/packages/schema/dist/routes.js.map +1 -0
  165. package/packages/schema/dist/schemas.d.ts +297 -0
  166. package/packages/schema/dist/schemas.js +243 -0
  167. package/packages/schema/dist/schemas.js.map +1 -0
  168. package/packages/schema/dist/textFiles.d.ts +5 -0
  169. package/packages/schema/dist/textFiles.js +66 -0
  170. package/packages/schema/dist/textFiles.js.map +1 -0
  171. package/packages/schema/package.json +26 -0
  172. package/packages/schema/src/ark.ts +29 -0
  173. package/packages/schema/src/index.ts +5 -0
  174. package/packages/schema/src/routes.ts +22 -0
  175. package/packages/schema/src/schemas.test.ts +123 -0
  176. package/packages/schema/src/schemas.ts +287 -0
  177. package/packages/schema/src/textFiles.test.ts +23 -0
  178. package/packages/schema/src/textFiles.ts +66 -0
  179. package/packages/schema/tsconfig.json +15 -0
  180. package/pilothub +46 -0
  181. package/playwright.config.ts +33 -0
  182. package/public/.well-known/pilothub.json +6 -0
  183. package/public/api/v1/openapi.json +379 -0
  184. package/public/favicon.ico +0 -0
  185. package/public/logo192.png +0 -0
  186. package/public/logo512.png +0 -0
  187. package/public/manifest.json +25 -0
  188. package/public/og.png +0 -0
  189. package/public/og.svg +98 -0
  190. package/public/pilot-logo.png +0 -0
  191. package/public/pilot-mark.png +0 -0
  192. package/public/robots.txt +3 -0
  193. package/public/tanstack-circle-logo.png +0 -0
  194. package/public/tanstack-word-logo-white.svg +1 -0
  195. package/scripts/check-peer-deps.ts +56 -0
  196. package/scripts/docs-list.ts +148 -0
  197. package/scripts/run-playwright-local.sh +14 -0
  198. package/server/og/fetchSkillOgMeta.ts +27 -0
  199. package/server/og/fetchSoulOgMeta.ts +27 -0
  200. package/server/og/ogAssets.ts +80 -0
  201. package/server/og/skillOgSvg.test.ts +59 -0
  202. package/server/og/skillOgSvg.ts +258 -0
  203. package/server/og/soulOgSvg.ts +209 -0
  204. package/server/routes/og/skill.png.ts +103 -0
  205. package/server/routes/og/soul.png.ts +111 -0
  206. package/src/__tests__/skill-detail-page.test.tsx +86 -0
  207. package/src/__tests__/skills-index.test.tsx +145 -0
  208. package/src/__tests__/upload.route.test.tsx +228 -0
  209. package/src/components/AppProviders.tsx +19 -0
  210. package/src/components/ClientOnly.tsx +18 -0
  211. package/src/components/Footer.tsx +29 -0
  212. package/src/components/Header.tsx +295 -0
  213. package/src/components/InstallSwitcher.tsx +53 -0
  214. package/src/components/SkillCard.tsx +36 -0
  215. package/src/components/SkillDetailPage.tsx +817 -0
  216. package/src/components/SkillDiffCard.tsx +485 -0
  217. package/src/components/SoulCard.tsx +19 -0
  218. package/src/components/SoulDetailPage.tsx +263 -0
  219. package/src/components/UserBootstrap.tsx +18 -0
  220. package/src/components/ui/dropdown-menu.tsx +67 -0
  221. package/src/components/ui/toggle-group.tsx +35 -0
  222. package/src/convex/client.ts +3 -0
  223. package/src/lib/badges.ts +29 -0
  224. package/src/lib/diffing.test.ts +163 -0
  225. package/src/lib/diffing.ts +106 -0
  226. package/src/lib/gravatar.test.ts +9 -0
  227. package/src/lib/gravatar.ts +158 -0
  228. package/src/lib/og.test.ts +142 -0
  229. package/src/lib/og.ts +156 -0
  230. package/src/lib/publicUser.ts +39 -0
  231. package/src/lib/roles.ts +19 -0
  232. package/src/lib/site.test.ts +130 -0
  233. package/src/lib/site.ts +84 -0
  234. package/src/lib/theme-transition.test.ts +134 -0
  235. package/src/lib/theme-transition.ts +134 -0
  236. package/src/lib/theme.test.tsx +88 -0
  237. package/src/lib/theme.ts +43 -0
  238. package/src/lib/uploadFiles.jsdom.test.ts +33 -0
  239. package/src/lib/uploadFiles.test.ts +123 -0
  240. package/src/lib/uploadFiles.ts +245 -0
  241. package/src/lib/uploadUtils.test.ts +78 -0
  242. package/src/lib/uploadUtils.ts +93 -0
  243. package/src/lib/useAuthStatus.ts +12 -0
  244. package/src/lib/utils.test.ts +9 -0
  245. package/src/lib/utils.ts +6 -0
  246. package/src/logo.svg +12 -0
  247. package/src/routeTree.gen.ts +345 -0
  248. package/src/router.tsx +17 -0
  249. package/src/routes/$owner/$slug.tsx +55 -0
  250. package/src/routes/__root.tsx +136 -0
  251. package/src/routes/admin.tsx +11 -0
  252. package/src/routes/cli/auth.tsx +168 -0
  253. package/src/routes/dashboard.tsx +97 -0
  254. package/src/routes/import.tsx +415 -0
  255. package/src/routes/index.tsx +252 -0
  256. package/src/routes/management.tsx +529 -0
  257. package/src/routes/settings.tsx +203 -0
  258. package/src/routes/skills/index.tsx +422 -0
  259. package/src/routes/souls/$slug.tsx +55 -0
  260. package/src/routes/souls/index.tsx +243 -0
  261. package/src/routes/stars.tsx +68 -0
  262. package/src/routes/u/$handle.tsx +307 -0
  263. package/src/routes/upload/utils.ts +81 -0
  264. package/src/routes/upload.tsx +499 -0
  265. package/src/styles.css +2718 -0
  266. package/tsconfig.json +24 -0
  267. package/tsconfig.oxlint.json +16 -0
  268. package/vercel.json +8 -0
  269. package/vite.config.ts +48 -0
  270. package/vitest.config.ts +47 -0
  271. package/vitest.e2e.config.ts +11 -0
  272. package/vitest.setup.ts +1 -0
@@ -0,0 +1,840 @@
1
+ import { ConvexError, v } from 'convex/values'
2
+ import { internal } from './_generated/api'
3
+ import type { Doc, Id } from './_generated/dataModel'
4
+ import type { ActionCtx } from './_generated/server'
5
+ import { action, internalAction, internalMutation, internalQuery } from './_generated/server'
6
+ import { assertRole, requireUserFromAction } from './lib/access'
7
+ import { buildSkillSummaryBackfillPatch, type ParsedSkillData } from './lib/skillBackfill'
8
+ import { hashSkillFiles } from './lib/skills'
9
+
10
+ const DEFAULT_BATCH_SIZE = 50
11
+ const MAX_BATCH_SIZE = 200
12
+ const DEFAULT_MAX_BATCHES = 20
13
+ const MAX_MAX_BATCHES = 200
14
+
15
+ type BackfillStats = {
16
+ skillsScanned: number
17
+ skillsPatched: number
18
+ versionsPatched: number
19
+ missingLatestVersion: number
20
+ missingReadme: number
21
+ missingStorageBlob: number
22
+ }
23
+
24
+ type BackfillPageItem =
25
+ | {
26
+ kind: 'ok'
27
+ skillId: Id<'skills'>
28
+ versionId: Id<'skillVersions'>
29
+ skillSummary: Doc<'skills'>['summary']
30
+ versionParsed: Doc<'skillVersions'>['parsed']
31
+ readmeStorageId: Id<'_storage'>
32
+ }
33
+ | { kind: 'missingLatestVersion'; skillId: Id<'skills'> }
34
+ | { kind: 'missingVersionDoc'; skillId: Id<'skills'>; versionId: Id<'skillVersions'> }
35
+ | { kind: 'missingReadme'; skillId: Id<'skills'>; versionId: Id<'skillVersions'> }
36
+
37
+ type BackfillPageResult = {
38
+ items: BackfillPageItem[]
39
+ cursor: string | null
40
+ isDone: boolean
41
+ }
42
+
43
+ export const getSkillBackfillPageInternal = internalQuery({
44
+ args: {
45
+ cursor: v.optional(v.string()),
46
+ batchSize: v.optional(v.number()),
47
+ },
48
+ handler: async (ctx, args): Promise<BackfillPageResult> => {
49
+ const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
50
+ const { page, isDone, continueCursor } = await ctx.db
51
+ .query('skills')
52
+ .order('asc')
53
+ .paginate({ cursor: args.cursor ?? null, numItems: batchSize })
54
+
55
+ const items: BackfillPageItem[] = []
56
+ for (const skill of page) {
57
+ if (!skill.latestVersionId) {
58
+ items.push({ kind: 'missingLatestVersion', skillId: skill._id })
59
+ continue
60
+ }
61
+
62
+ const version = await ctx.db.get(skill.latestVersionId)
63
+ if (!version) {
64
+ items.push({
65
+ kind: 'missingVersionDoc',
66
+ skillId: skill._id,
67
+ versionId: skill.latestVersionId,
68
+ })
69
+ continue
70
+ }
71
+
72
+ const readmeFile = version.files.find(
73
+ (file) => file.path.toLowerCase() === 'skill.md' || file.path.toLowerCase() === 'skills.md',
74
+ )
75
+ if (!readmeFile) {
76
+ items.push({ kind: 'missingReadme', skillId: skill._id, versionId: version._id })
77
+ continue
78
+ }
79
+
80
+ items.push({
81
+ kind: 'ok',
82
+ skillId: skill._id,
83
+ versionId: version._id,
84
+ skillSummary: skill.summary,
85
+ versionParsed: version.parsed,
86
+ readmeStorageId: readmeFile.storageId,
87
+ })
88
+ }
89
+
90
+ return { items, cursor: continueCursor, isDone }
91
+ },
92
+ })
93
+
94
+ export const applySkillBackfillPatchInternal = internalMutation({
95
+ args: {
96
+ skillId: v.id('skills'),
97
+ versionId: v.id('skillVersions'),
98
+ summary: v.optional(v.string()),
99
+ parsed: v.optional(
100
+ v.object({
101
+ frontmatter: v.record(v.string(), v.any()),
102
+ metadata: v.optional(v.any()),
103
+ pilotbot: v.optional(v.any()),
104
+ }),
105
+ ),
106
+ },
107
+ handler: async (ctx, args) => {
108
+ const now = Date.now()
109
+ if (typeof args.summary === 'string') {
110
+ await ctx.db.patch(args.skillId, { summary: args.summary, updatedAt: now })
111
+ }
112
+ if (args.parsed) {
113
+ await ctx.db.patch(args.versionId, { parsed: args.parsed })
114
+ }
115
+ return { ok: true as const }
116
+ },
117
+ })
118
+
119
+ export type BackfillActionArgs = {
120
+ dryRun?: boolean
121
+ batchSize?: number
122
+ maxBatches?: number
123
+ }
124
+
125
+ export type BackfillActionResult = { ok: true; stats: BackfillStats }
126
+
127
+ export async function backfillSkillSummariesInternalHandler(
128
+ ctx: ActionCtx,
129
+ args: BackfillActionArgs,
130
+ ): Promise<BackfillActionResult> {
131
+ const dryRun = Boolean(args.dryRun)
132
+ const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
133
+ const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES)
134
+
135
+ const totals: BackfillStats = {
136
+ skillsScanned: 0,
137
+ skillsPatched: 0,
138
+ versionsPatched: 0,
139
+ missingLatestVersion: 0,
140
+ missingReadme: 0,
141
+ missingStorageBlob: 0,
142
+ }
143
+
144
+ let cursor: string | null = null
145
+ let isDone = false
146
+
147
+ for (let i = 0; i < maxBatches; i++) {
148
+ const page = (await ctx.runQuery(internal.maintenance.getSkillBackfillPageInternal, {
149
+ cursor: cursor ?? undefined,
150
+ batchSize,
151
+ })) as BackfillPageResult
152
+
153
+ cursor = page.cursor
154
+ isDone = page.isDone
155
+
156
+ for (const item of page.items) {
157
+ totals.skillsScanned++
158
+ if (item.kind === 'missingLatestVersion') {
159
+ totals.missingLatestVersion++
160
+ continue
161
+ }
162
+ if (item.kind === 'missingVersionDoc') {
163
+ totals.missingLatestVersion++
164
+ continue
165
+ }
166
+ if (item.kind === 'missingReadme') {
167
+ totals.missingReadme++
168
+ continue
169
+ }
170
+
171
+ const blob = await ctx.storage.get(item.readmeStorageId)
172
+ if (!blob) {
173
+ totals.missingStorageBlob++
174
+ continue
175
+ }
176
+
177
+ const readmeText = await blob.text()
178
+ const patch = buildSkillSummaryBackfillPatch({
179
+ readmeText,
180
+ currentSummary: item.skillSummary ?? undefined,
181
+ currentParsed: item.versionParsed as ParsedSkillData,
182
+ })
183
+
184
+ if (!patch.summary && !patch.parsed) continue
185
+ if (patch.summary) totals.skillsPatched++
186
+ if (patch.parsed) totals.versionsPatched++
187
+
188
+ if (dryRun) continue
189
+
190
+ await ctx.runMutation(internal.maintenance.applySkillBackfillPatchInternal, {
191
+ skillId: item.skillId,
192
+ versionId: item.versionId,
193
+ summary: patch.summary,
194
+ parsed: patch.parsed,
195
+ })
196
+ }
197
+
198
+ if (isDone) break
199
+ }
200
+
201
+ if (!isDone) {
202
+ throw new ConvexError('Backfill incomplete (maxBatches reached)')
203
+ }
204
+
205
+ return { ok: true as const, stats: totals }
206
+ }
207
+
208
+ export const backfillSkillSummariesInternal = internalAction({
209
+ args: {
210
+ dryRun: v.optional(v.boolean()),
211
+ batchSize: v.optional(v.number()),
212
+ maxBatches: v.optional(v.number()),
213
+ },
214
+ handler: backfillSkillSummariesInternalHandler,
215
+ })
216
+
217
+ export const backfillSkillSummaries: ReturnType<typeof action> = action({
218
+ args: {
219
+ dryRun: v.optional(v.boolean()),
220
+ batchSize: v.optional(v.number()),
221
+ maxBatches: v.optional(v.number()),
222
+ },
223
+ handler: async (ctx, args): Promise<BackfillActionResult> => {
224
+ const { user } = await requireUserFromAction(ctx)
225
+ assertRole(user, ['admin'])
226
+ return ctx.runAction(
227
+ internal.maintenance.backfillSkillSummariesInternal,
228
+ args,
229
+ ) as Promise<BackfillActionResult>
230
+ },
231
+ })
232
+
233
+ export const scheduleBackfillSkillSummaries: ReturnType<typeof action> = action({
234
+ args: { dryRun: v.optional(v.boolean()) },
235
+ handler: async (ctx, args) => {
236
+ const { user } = await requireUserFromAction(ctx)
237
+ assertRole(user, ['admin'])
238
+ await ctx.scheduler.runAfter(0, internal.maintenance.backfillSkillSummariesInternal, {
239
+ dryRun: Boolean(args.dryRun),
240
+ batchSize: DEFAULT_BATCH_SIZE,
241
+ maxBatches: DEFAULT_MAX_BATCHES,
242
+ })
243
+ return { ok: true as const }
244
+ },
245
+ })
246
+
247
+ type FingerprintBackfillStats = {
248
+ versionsScanned: number
249
+ versionsPatched: number
250
+ fingerprintsInserted: number
251
+ fingerprintMismatches: number
252
+ }
253
+
254
+ type FingerprintBackfillPageItem = {
255
+ skillId: Id<'skills'>
256
+ versionId: Id<'skillVersions'>
257
+ versionFingerprint?: string
258
+ files: Array<{ path: string; sha256: string }>
259
+ existingEntries: Array<{ id: Id<'skillVersionFingerprints'>; fingerprint: string }>
260
+ }
261
+
262
+ type FingerprintBackfillPageResult = {
263
+ items: FingerprintBackfillPageItem[]
264
+ cursor: string | null
265
+ isDone: boolean
266
+ }
267
+
268
+ type BadgeBackfillStats = {
269
+ skillsScanned: number
270
+ skillsPatched: number
271
+ highlightsPatched: number
272
+ }
273
+
274
+ type SkillBadgeTableBackfillStats = {
275
+ skillsScanned: number
276
+ recordsInserted: number
277
+ }
278
+
279
+ type BadgeBackfillPageItem = {
280
+ skillId: Id<'skills'>
281
+ ownerUserId: Id<'users'>
282
+ createdAt?: number
283
+ updatedAt?: number
284
+ batch?: string
285
+ badges?: Doc<'skills'>['badges']
286
+ }
287
+
288
+ type BadgeBackfillPageResult = {
289
+ items: BadgeBackfillPageItem[]
290
+ cursor: string | null
291
+ isDone: boolean
292
+ }
293
+
294
+ type BadgeKind = Doc<'skillBadges'>['kind']
295
+
296
+ export const getSkillFingerprintBackfillPageInternal = internalQuery({
297
+ args: {
298
+ cursor: v.optional(v.string()),
299
+ batchSize: v.optional(v.number()),
300
+ },
301
+ handler: async (ctx, args): Promise<FingerprintBackfillPageResult> => {
302
+ const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
303
+ const { page, isDone, continueCursor } = await ctx.db
304
+ .query('skillVersions')
305
+ .order('asc')
306
+ .paginate({ cursor: args.cursor ?? null, numItems: batchSize })
307
+
308
+ const items: FingerprintBackfillPageItem[] = []
309
+ for (const version of page) {
310
+ const existingEntries = await ctx.db
311
+ .query('skillVersionFingerprints')
312
+ .withIndex('by_version', (q) => q.eq('versionId', version._id))
313
+ .take(20)
314
+
315
+ const normalizedFiles = version.files.map((file) => ({
316
+ path: file.path,
317
+ sha256: file.sha256,
318
+ }))
319
+
320
+ const hasAnyEntry = existingEntries.length > 0
321
+ const entryFingerprints = new Set(existingEntries.map((entry) => entry.fingerprint))
322
+ const hasFingerprintMismatch =
323
+ typeof version.fingerprint === 'string' &&
324
+ hasAnyEntry &&
325
+ (entryFingerprints.size !== 1 || !entryFingerprints.has(version.fingerprint))
326
+ const needsFingerprintField = !version.fingerprint
327
+ const needsFingerprintEntry = !hasAnyEntry
328
+
329
+ if (!needsFingerprintField && !needsFingerprintEntry && !hasFingerprintMismatch) continue
330
+
331
+ items.push({
332
+ skillId: version.skillId,
333
+ versionId: version._id,
334
+ versionFingerprint: version.fingerprint ?? undefined,
335
+ files: normalizedFiles,
336
+ existingEntries: existingEntries.map((entry) => ({
337
+ id: entry._id,
338
+ fingerprint: entry.fingerprint,
339
+ })),
340
+ })
341
+ }
342
+
343
+ return { items, cursor: continueCursor, isDone }
344
+ },
345
+ })
346
+
347
+ export const applySkillFingerprintBackfillPatchInternal = internalMutation({
348
+ args: {
349
+ versionId: v.id('skillVersions'),
350
+ fingerprint: v.string(),
351
+ patchVersion: v.boolean(),
352
+ replaceEntries: v.boolean(),
353
+ existingEntryIds: v.optional(v.array(v.id('skillVersionFingerprints'))),
354
+ },
355
+ handler: async (ctx, args) => {
356
+ const version = await ctx.db.get(args.versionId)
357
+ if (!version) return { ok: false as const, reason: 'missingVersion' as const }
358
+
359
+ const now = Date.now()
360
+
361
+ if (args.patchVersion) {
362
+ await ctx.db.patch(version._id, { fingerprint: args.fingerprint })
363
+ }
364
+
365
+ if (args.replaceEntries) {
366
+ const existing = args.existingEntryIds ?? []
367
+ for (const id of existing) {
368
+ await ctx.db.delete(id)
369
+ }
370
+
371
+ await ctx.db.insert('skillVersionFingerprints', {
372
+ skillId: version.skillId,
373
+ versionId: version._id,
374
+ fingerprint: args.fingerprint,
375
+ createdAt: now,
376
+ })
377
+ }
378
+
379
+ return { ok: true as const }
380
+ },
381
+ })
382
+
383
+ export type FingerprintBackfillActionArgs = {
384
+ dryRun?: boolean
385
+ batchSize?: number
386
+ maxBatches?: number
387
+ }
388
+
389
+ export type FingerprintBackfillActionResult = { ok: true; stats: FingerprintBackfillStats }
390
+
391
+ export async function backfillSkillFingerprintsInternalHandler(
392
+ ctx: ActionCtx,
393
+ args: FingerprintBackfillActionArgs,
394
+ ): Promise<FingerprintBackfillActionResult> {
395
+ const dryRun = Boolean(args.dryRun)
396
+ const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
397
+ const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES)
398
+
399
+ const totals: FingerprintBackfillStats = {
400
+ versionsScanned: 0,
401
+ versionsPatched: 0,
402
+ fingerprintsInserted: 0,
403
+ fingerprintMismatches: 0,
404
+ }
405
+
406
+ let cursor: string | null = null
407
+ let isDone = false
408
+
409
+ for (let i = 0; i < maxBatches; i++) {
410
+ const page = (await ctx.runQuery(internal.maintenance.getSkillFingerprintBackfillPageInternal, {
411
+ cursor: cursor ?? undefined,
412
+ batchSize,
413
+ })) as FingerprintBackfillPageResult
414
+
415
+ cursor = page.cursor
416
+ isDone = page.isDone
417
+
418
+ for (const item of page.items) {
419
+ totals.versionsScanned++
420
+
421
+ const fingerprint = await hashSkillFiles(item.files)
422
+
423
+ const existingFingerprints = new Set(item.existingEntries.map((entry) => entry.fingerprint))
424
+ const hasAnyEntry = item.existingEntries.length > 0
425
+ const entryIsCorrect =
426
+ hasAnyEntry && existingFingerprints.size === 1 && existingFingerprints.has(fingerprint)
427
+ const versionFingerprintIsCorrect = item.versionFingerprint === fingerprint
428
+
429
+ if (hasAnyEntry && !entryIsCorrect) totals.fingerprintMismatches++
430
+
431
+ const shouldPatchVersion = !versionFingerprintIsCorrect
432
+ const shouldReplaceEntries = !entryIsCorrect
433
+ if (!shouldPatchVersion && !shouldReplaceEntries) continue
434
+
435
+ if (shouldPatchVersion) totals.versionsPatched++
436
+ if (shouldReplaceEntries) totals.fingerprintsInserted++
437
+
438
+ if (dryRun) continue
439
+
440
+ await ctx.runMutation(internal.maintenance.applySkillFingerprintBackfillPatchInternal, {
441
+ versionId: item.versionId,
442
+ fingerprint,
443
+ patchVersion: shouldPatchVersion,
444
+ replaceEntries: shouldReplaceEntries,
445
+ existingEntryIds: shouldReplaceEntries ? item.existingEntries.map((entry) => entry.id) : [],
446
+ })
447
+ }
448
+
449
+ if (isDone) break
450
+ }
451
+
452
+ if (!isDone) {
453
+ throw new ConvexError('Backfill incomplete (maxBatches reached)')
454
+ }
455
+
456
+ return { ok: true as const, stats: totals }
457
+ }
458
+
459
+ export const backfillSkillFingerprintsInternal = internalAction({
460
+ args: {
461
+ dryRun: v.optional(v.boolean()),
462
+ batchSize: v.optional(v.number()),
463
+ maxBatches: v.optional(v.number()),
464
+ },
465
+ handler: backfillSkillFingerprintsInternalHandler,
466
+ })
467
+
468
+ export const backfillSkillFingerprints: ReturnType<typeof action> = action({
469
+ args: {
470
+ dryRun: v.optional(v.boolean()),
471
+ batchSize: v.optional(v.number()),
472
+ maxBatches: v.optional(v.number()),
473
+ },
474
+ handler: async (ctx, args): Promise<FingerprintBackfillActionResult> => {
475
+ const { user } = await requireUserFromAction(ctx)
476
+ assertRole(user, ['admin'])
477
+ return ctx.runAction(
478
+ internal.maintenance.backfillSkillFingerprintsInternal,
479
+ args,
480
+ ) as Promise<FingerprintBackfillActionResult>
481
+ },
482
+ })
483
+
484
+ export const scheduleBackfillSkillFingerprints: ReturnType<typeof action> = action({
485
+ args: { dryRun: v.optional(v.boolean()) },
486
+ handler: async (ctx, args) => {
487
+ const { user } = await requireUserFromAction(ctx)
488
+ assertRole(user, ['admin'])
489
+ await ctx.scheduler.runAfter(0, internal.maintenance.backfillSkillFingerprintsInternal, {
490
+ dryRun: Boolean(args.dryRun),
491
+ batchSize: DEFAULT_BATCH_SIZE,
492
+ maxBatches: DEFAULT_MAX_BATCHES,
493
+ })
494
+ return { ok: true as const }
495
+ },
496
+ })
497
+
498
+ export const getSkillBadgeBackfillPageInternal = internalQuery({
499
+ args: {
500
+ cursor: v.optional(v.string()),
501
+ batchSize: v.optional(v.number()),
502
+ },
503
+ handler: async (ctx, args): Promise<BadgeBackfillPageResult> => {
504
+ const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
505
+ const { page, isDone, continueCursor } = await ctx.db
506
+ .query('skills')
507
+ .order('asc')
508
+ .paginate({ cursor: args.cursor ?? null, numItems: batchSize })
509
+
510
+ const items: BadgeBackfillPageItem[] = page.map((skill) => ({
511
+ skillId: skill._id,
512
+ ownerUserId: skill.ownerUserId,
513
+ createdAt: skill.createdAt ?? undefined,
514
+ updatedAt: skill.updatedAt ?? undefined,
515
+ batch: skill.batch ?? undefined,
516
+ badges: skill.badges ?? undefined,
517
+ }))
518
+
519
+ return { items, cursor: continueCursor, isDone }
520
+ },
521
+ })
522
+
523
+ export const applySkillBadgeBackfillPatchInternal = internalMutation({
524
+ args: {
525
+ skillId: v.id('skills'),
526
+ badges: v.optional(
527
+ v.object({
528
+ redactionApproved: v.optional(
529
+ v.object({
530
+ byUserId: v.id('users'),
531
+ at: v.number(),
532
+ }),
533
+ ),
534
+ highlighted: v.optional(
535
+ v.object({
536
+ byUserId: v.id('users'),
537
+ at: v.number(),
538
+ }),
539
+ ),
540
+ official: v.optional(
541
+ v.object({
542
+ byUserId: v.id('users'),
543
+ at: v.number(),
544
+ }),
545
+ ),
546
+ deprecated: v.optional(
547
+ v.object({
548
+ byUserId: v.id('users'),
549
+ at: v.number(),
550
+ }),
551
+ ),
552
+ }),
553
+ ),
554
+ },
555
+ handler: async (ctx, args) => {
556
+ await ctx.db.patch(args.skillId, { badges: args.badges ?? undefined, updatedAt: Date.now() })
557
+ return { ok: true as const }
558
+ },
559
+ })
560
+
561
+ export const upsertSkillBadgeRecordInternal = internalMutation({
562
+ args: {
563
+ skillId: v.id('skills'),
564
+ kind: v.union(
565
+ v.literal('highlighted'),
566
+ v.literal('official'),
567
+ v.literal('deprecated'),
568
+ v.literal('redactionApproved'),
569
+ ),
570
+ byUserId: v.id('users'),
571
+ at: v.number(),
572
+ },
573
+ handler: async (ctx, args) => {
574
+ const existing = await ctx.db
575
+ .query('skillBadges')
576
+ .withIndex('by_skill_kind', (q) => q.eq('skillId', args.skillId).eq('kind', args.kind))
577
+ .unique()
578
+ if (existing) return { inserted: false as const }
579
+ await ctx.db.insert('skillBadges', {
580
+ skillId: args.skillId,
581
+ kind: args.kind,
582
+ byUserId: args.byUserId,
583
+ at: args.at,
584
+ })
585
+ return { inserted: true as const }
586
+ },
587
+ })
588
+
589
+ export type BadgeBackfillActionArgs = {
590
+ dryRun?: boolean
591
+ batchSize?: number
592
+ maxBatches?: number
593
+ }
594
+
595
+ export type BadgeBackfillActionResult = { ok: true; stats: BadgeBackfillStats }
596
+
597
+ export async function backfillSkillBadgesInternalHandler(
598
+ ctx: ActionCtx,
599
+ args: BadgeBackfillActionArgs,
600
+ ): Promise<BadgeBackfillActionResult> {
601
+ const dryRun = Boolean(args.dryRun)
602
+ const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
603
+ const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES)
604
+
605
+ const totals: BadgeBackfillStats = {
606
+ skillsScanned: 0,
607
+ skillsPatched: 0,
608
+ highlightsPatched: 0,
609
+ }
610
+
611
+ let cursor: string | null = null
612
+ let isDone = false
613
+
614
+ for (let i = 0; i < maxBatches; i++) {
615
+ const page = (await ctx.runQuery(internal.maintenance.getSkillBadgeBackfillPageInternal, {
616
+ cursor: cursor ?? undefined,
617
+ batchSize,
618
+ })) as BadgeBackfillPageResult
619
+
620
+ cursor = page.cursor
621
+ isDone = page.isDone
622
+
623
+ for (const item of page.items) {
624
+ totals.skillsScanned++
625
+
626
+ const shouldHighlight = item.batch === 'highlighted' && !item.badges?.highlighted
627
+ if (!shouldHighlight) continue
628
+
629
+ totals.skillsPatched++
630
+ totals.highlightsPatched++
631
+
632
+ if (dryRun) continue
633
+
634
+ const at = item.updatedAt ?? item.createdAt ?? Date.now()
635
+ await ctx.runMutation(internal.maintenance.applySkillBadgeBackfillPatchInternal, {
636
+ skillId: item.skillId,
637
+ badges: {
638
+ ...item.badges,
639
+ highlighted: {
640
+ byUserId: item.ownerUserId,
641
+ at,
642
+ },
643
+ },
644
+ })
645
+ }
646
+
647
+ if (isDone) break
648
+ }
649
+
650
+ if (!isDone) {
651
+ throw new ConvexError('Backfill incomplete (maxBatches reached)')
652
+ }
653
+
654
+ return { ok: true as const, stats: totals }
655
+ }
656
+
657
+ export const backfillSkillBadgesInternal = internalAction({
658
+ args: {
659
+ dryRun: v.optional(v.boolean()),
660
+ batchSize: v.optional(v.number()),
661
+ maxBatches: v.optional(v.number()),
662
+ },
663
+ handler: backfillSkillBadgesInternalHandler,
664
+ })
665
+
666
+ export const backfillSkillBadges: ReturnType<typeof action> = action({
667
+ args: {
668
+ dryRun: v.optional(v.boolean()),
669
+ batchSize: v.optional(v.number()),
670
+ maxBatches: v.optional(v.number()),
671
+ },
672
+ handler: async (ctx, args): Promise<BadgeBackfillActionResult> => {
673
+ const { user } = await requireUserFromAction(ctx)
674
+ assertRole(user, ['admin'])
675
+ return ctx.runAction(
676
+ internal.maintenance.backfillSkillBadgesInternal,
677
+ args,
678
+ ) as Promise<BadgeBackfillActionResult>
679
+ },
680
+ })
681
+
682
+ export const scheduleBackfillSkillBadges: ReturnType<typeof action> = action({
683
+ args: { dryRun: v.optional(v.boolean()) },
684
+ handler: async (ctx, args) => {
685
+ const { user } = await requireUserFromAction(ctx)
686
+ assertRole(user, ['admin'])
687
+ await ctx.scheduler.runAfter(0, internal.maintenance.backfillSkillBadgesInternal, {
688
+ dryRun: Boolean(args.dryRun),
689
+ batchSize: DEFAULT_BATCH_SIZE,
690
+ maxBatches: DEFAULT_MAX_BATCHES,
691
+ })
692
+ return { ok: true as const }
693
+ },
694
+ })
695
+
696
+ export type SkillBadgeTableBackfillActionResult = {
697
+ ok: true
698
+ stats: SkillBadgeTableBackfillStats
699
+ }
700
+
701
+ export async function backfillSkillBadgeTableInternalHandler(
702
+ ctx: ActionCtx,
703
+ args: BadgeBackfillActionArgs,
704
+ ): Promise<SkillBadgeTableBackfillActionResult> {
705
+ const dryRun = Boolean(args.dryRun)
706
+ const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
707
+ const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES)
708
+
709
+ const totals: SkillBadgeTableBackfillStats = {
710
+ skillsScanned: 0,
711
+ recordsInserted: 0,
712
+ }
713
+
714
+ let cursor: string | null = null
715
+ let isDone = false
716
+
717
+ for (let i = 0; i < maxBatches; i++) {
718
+ const page = (await ctx.runQuery(internal.maintenance.getSkillBadgeBackfillPageInternal, {
719
+ cursor: cursor ?? undefined,
720
+ batchSize,
721
+ })) as BadgeBackfillPageResult
722
+
723
+ cursor = page.cursor
724
+ isDone = page.isDone
725
+
726
+ for (const item of page.items) {
727
+ totals.skillsScanned++
728
+ const badges = item.badges ?? {}
729
+ const entries: Array<{ kind: BadgeKind; byUserId: Id<'users'>; at: number }> = []
730
+
731
+ if (badges.redactionApproved) {
732
+ entries.push({
733
+ kind: 'redactionApproved',
734
+ byUserId: badges.redactionApproved.byUserId,
735
+ at: badges.redactionApproved.at,
736
+ })
737
+ }
738
+
739
+ if (badges.official) {
740
+ entries.push({
741
+ kind: 'official',
742
+ byUserId: badges.official.byUserId,
743
+ at: badges.official.at,
744
+ })
745
+ }
746
+
747
+ if (badges.deprecated) {
748
+ entries.push({
749
+ kind: 'deprecated',
750
+ byUserId: badges.deprecated.byUserId,
751
+ at: badges.deprecated.at,
752
+ })
753
+ }
754
+
755
+ const highlighted =
756
+ badges.highlighted ??
757
+ (item.batch === 'highlighted'
758
+ ? {
759
+ byUserId: item.ownerUserId,
760
+ at: item.updatedAt ?? item.createdAt ?? Date.now(),
761
+ }
762
+ : undefined)
763
+
764
+ if (highlighted) {
765
+ entries.push({
766
+ kind: 'highlighted',
767
+ byUserId: highlighted.byUserId,
768
+ at: highlighted.at,
769
+ })
770
+ }
771
+
772
+ if (dryRun) continue
773
+
774
+ for (const entry of entries) {
775
+ const result = await ctx.runMutation(internal.maintenance.upsertSkillBadgeRecordInternal, {
776
+ skillId: item.skillId,
777
+ kind: entry.kind,
778
+ byUserId: entry.byUserId,
779
+ at: entry.at,
780
+ })
781
+ if (result.inserted) {
782
+ totals.recordsInserted++
783
+ }
784
+ }
785
+ }
786
+
787
+ if (isDone) break
788
+ }
789
+
790
+ if (!isDone) {
791
+ throw new ConvexError('Backfill incomplete (maxBatches reached)')
792
+ }
793
+
794
+ return { ok: true as const, stats: totals }
795
+ }
796
+
797
+ export const backfillSkillBadgeTableInternal = internalAction({
798
+ args: {
799
+ dryRun: v.optional(v.boolean()),
800
+ batchSize: v.optional(v.number()),
801
+ maxBatches: v.optional(v.number()),
802
+ },
803
+ handler: backfillSkillBadgeTableInternalHandler,
804
+ })
805
+
806
+ export const backfillSkillBadgeTable: ReturnType<typeof action> = action({
807
+ args: {
808
+ dryRun: v.optional(v.boolean()),
809
+ batchSize: v.optional(v.number()),
810
+ maxBatches: v.optional(v.number()),
811
+ },
812
+ handler: async (ctx, args): Promise<SkillBadgeTableBackfillActionResult> => {
813
+ const { user } = await requireUserFromAction(ctx)
814
+ assertRole(user, ['admin'])
815
+ return ctx.runAction(
816
+ internal.maintenance.backfillSkillBadgeTableInternal,
817
+ args,
818
+ ) as Promise<SkillBadgeTableBackfillActionResult>
819
+ },
820
+ })
821
+
822
+ export const scheduleBackfillSkillBadgeTable: ReturnType<typeof action> = action({
823
+ args: { dryRun: v.optional(v.boolean()) },
824
+ handler: async (ctx, args) => {
825
+ const { user } = await requireUserFromAction(ctx)
826
+ assertRole(user, ['admin'])
827
+ await ctx.scheduler.runAfter(0, internal.maintenance.backfillSkillBadgeTableInternal, {
828
+ dryRun: Boolean(args.dryRun),
829
+ batchSize: DEFAULT_BATCH_SIZE,
830
+ maxBatches: DEFAULT_MAX_BATCHES,
831
+ })
832
+ return { ok: true as const }
833
+ },
834
+ })
835
+
836
+ function clampInt(value: number, min: number, max: number) {
837
+ const rounded = Math.trunc(value)
838
+ if (!Number.isFinite(rounded)) return min
839
+ return Math.min(max, Math.max(min, rounded))
840
+ }