pilothub 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.local.example +19 -0
- package/.github/workflows/ci.yml +40 -0
- package/.oxlintrc.json +3 -0
- package/AGENTS.md +45 -0
- package/CHANGELOG.md +138 -0
- package/DEPRECATIONS.md +7 -0
- package/LICENSE +21 -0
- package/README.md +150 -0
- package/biome.json +41 -0
- package/convex/_generated/api.d.ts +153 -0
- package/convex/_generated/api.js +23 -0
- package/convex/_generated/dataModel.d.ts +60 -0
- package/convex/_generated/server.d.ts +143 -0
- package/convex/_generated/server.js +93 -0
- package/convex/auth.config.ts +8 -0
- package/convex/auth.ts +19 -0
- package/convex/comments.ts +88 -0
- package/convex/crons.ts +34 -0
- package/convex/devSeed.ts +459 -0
- package/convex/devSeedExtra.ts +541 -0
- package/convex/downloads.ts +78 -0
- package/convex/githubBackups.ts +170 -0
- package/convex/githubBackupsNode.ts +183 -0
- package/convex/githubImport.ts +317 -0
- package/convex/githubSoulBackups.ts +170 -0
- package/convex/githubSoulBackupsNode.ts +186 -0
- package/convex/http.ts +194 -0
- package/convex/httpApi.handlers.test.ts +488 -0
- package/convex/httpApi.test.ts +70 -0
- package/convex/httpApi.ts +305 -0
- package/convex/httpApiV1.handlers.test.ts +584 -0
- package/convex/httpApiV1.ts +1172 -0
- package/convex/leaderboards.ts +39 -0
- package/convex/lib/access.ts +36 -0
- package/convex/lib/apiTokenAuth.ts +36 -0
- package/convex/lib/badges.ts +50 -0
- package/convex/lib/changelog.test.ts +34 -0
- package/convex/lib/changelog.ts +278 -0
- package/convex/lib/embeddings.ts +38 -0
- package/convex/lib/githubBackup.ts +443 -0
- package/convex/lib/githubImport.test.ts +247 -0
- package/convex/lib/githubImport.ts +425 -0
- package/convex/lib/githubSoulBackup.ts +443 -0
- package/convex/lib/leaderboards.ts +103 -0
- package/convex/lib/moderation.ts +42 -0
- package/convex/lib/public.ts +89 -0
- package/convex/lib/searchText.test.ts +46 -0
- package/convex/lib/searchText.ts +27 -0
- package/convex/lib/skillBackfill.test.ts +34 -0
- package/convex/lib/skillBackfill.ts +67 -0
- package/convex/lib/skillPublish.test.ts +28 -0
- package/convex/lib/skillPublish.ts +284 -0
- package/convex/lib/skillStats.ts +80 -0
- package/convex/lib/skills.test.ts +197 -0
- package/convex/lib/skills.ts +273 -0
- package/convex/lib/soulChangelog.ts +273 -0
- package/convex/lib/soulPublish.ts +236 -0
- package/convex/lib/tokens.test.ts +33 -0
- package/convex/lib/tokens.ts +51 -0
- package/convex/lib/webhooks.test.ts +91 -0
- package/convex/lib/webhooks.ts +112 -0
- package/convex/maintenance.test.ts +270 -0
- package/convex/maintenance.ts +840 -0
- package/convex/rateLimits.ts +50 -0
- package/convex/schema.ts +472 -0
- package/convex/search.test.ts +12 -0
- package/convex/search.ts +254 -0
- package/convex/seed.test.ts +37 -0
- package/convex/seed.ts +254 -0
- package/convex/seedSouls.ts +111 -0
- package/convex/skillStatEvents.ts +568 -0
- package/convex/skills.ts +1606 -0
- package/convex/soulComments.ts +88 -0
- package/convex/soulDownloads.ts +14 -0
- package/convex/soulStars.ts +71 -0
- package/convex/souls.ts +570 -0
- package/convex/stars.ts +108 -0
- package/convex/statsMaintenance.ts +205 -0
- package/convex/telemetry.ts +434 -0
- package/convex/tokens.ts +88 -0
- package/convex/tsconfig.json +7 -0
- package/convex/uploads.ts +20 -0
- package/convex/users.ts +122 -0
- package/convex/webhooks.ts +50 -0
- package/convex.json +3 -0
- package/docs/README.md +32 -0
- package/docs/api.md +51 -0
- package/docs/architecture.md +61 -0
- package/docs/auth.md +54 -0
- package/docs/cli.md +117 -0
- package/docs/deploy.md +78 -0
- package/docs/diffing.md +84 -0
- package/docs/github-import.md +171 -0
- package/docs/http-api.md +187 -0
- package/docs/manual-testing.md +64 -0
- package/docs/mintlify.md +43 -0
- package/docs/quickstart.md +120 -0
- package/docs/skill-format.md +58 -0
- package/docs/soul-format.md +37 -0
- package/docs/spec.md +177 -0
- package/docs/telemetry.md +91 -0
- package/docs/troubleshooting.md +49 -0
- package/docs/webhook.md +51 -0
- package/e2e/menu-smoke.pw.test.ts +49 -0
- package/e2e/pilothub.e2e.test.ts +494 -0
- package/e2e/search-exact.pw.test.ts +97 -0
- package/package.json +84 -0
- package/packages/pilothub/LICENSE +22 -0
- package/packages/pilothub/README.md +57 -0
- package/packages/pilothub/bin/pilothub.js +2 -0
- package/packages/pilothub/package.json +41 -0
- package/packages/pilothub/src/browserAuth.test.ts +96 -0
- package/packages/pilothub/src/browserAuth.ts +174 -0
- package/packages/pilothub/src/cli/buildInfo.ts +94 -0
- package/packages/pilothub/src/cli/commands/auth.ts +97 -0
- package/packages/pilothub/src/cli/commands/delete.test.ts +73 -0
- package/packages/pilothub/src/cli/commands/delete.ts +83 -0
- package/packages/pilothub/src/cli/commands/publish.test.ts +122 -0
- package/packages/pilothub/src/cli/commands/publish.ts +108 -0
- package/packages/pilothub/src/cli/commands/skills.test.ts +191 -0
- package/packages/pilothub/src/cli/commands/skills.ts +380 -0
- package/packages/pilothub/src/cli/commands/star.ts +46 -0
- package/packages/pilothub/src/cli/commands/sync.test.ts +310 -0
- package/packages/pilothub/src/cli/commands/sync.ts +200 -0
- package/packages/pilothub/src/cli/commands/syncHelpers.test.ts +26 -0
- package/packages/pilothub/src/cli/commands/syncHelpers.ts +427 -0
- package/packages/pilothub/src/cli/commands/syncTypes.ts +27 -0
- package/packages/pilothub/src/cli/commands/unstar.ts +48 -0
- package/packages/pilothub/src/cli/helpStyle.ts +45 -0
- package/packages/pilothub/src/cli/pilotbotConfig.test.ts +159 -0
- package/packages/pilothub/src/cli/pilotbotConfig.ts +147 -0
- package/packages/pilothub/src/cli/registry.test.ts +63 -0
- package/packages/pilothub/src/cli/registry.ts +43 -0
- package/packages/pilothub/src/cli/scanSkills.test.ts +64 -0
- package/packages/pilothub/src/cli/scanSkills.ts +84 -0
- package/packages/pilothub/src/cli/slug.ts +16 -0
- package/packages/pilothub/src/cli/types.ts +12 -0
- package/packages/pilothub/src/cli/ui.ts +75 -0
- package/packages/pilothub/src/cli.ts +311 -0
- package/packages/pilothub/src/config.ts +36 -0
- package/packages/pilothub/src/discovery.test.ts +75 -0
- package/packages/pilothub/src/discovery.ts +19 -0
- package/packages/pilothub/src/http.test.ts +156 -0
- package/packages/pilothub/src/http.ts +301 -0
- package/packages/pilothub/src/schema/ark.ts +29 -0
- package/packages/pilothub/src/schema/index.ts +5 -0
- package/packages/pilothub/src/schema/routes.ts +22 -0
- package/packages/pilothub/src/schema/schemas.ts +260 -0
- package/packages/pilothub/src/schema/textFiles.test.ts +23 -0
- package/packages/pilothub/src/schema/textFiles.ts +66 -0
- package/packages/pilothub/src/skills.test.ts +191 -0
- package/packages/pilothub/src/skills.ts +172 -0
- package/packages/pilothub/src/types.ts +10 -0
- package/packages/pilothub/tsconfig.json +14 -0
- package/packages/schema/README.md +3 -0
- package/packages/schema/dist/ark.d.ts +4 -0
- package/packages/schema/dist/ark.js +26 -0
- package/packages/schema/dist/ark.js.map +1 -0
- package/packages/schema/dist/index.d.ts +5 -0
- package/packages/schema/dist/index.js +5 -0
- package/packages/schema/dist/index.js.map +1 -0
- package/packages/schema/dist/routes.d.ts +21 -0
- package/packages/schema/dist/routes.js +22 -0
- package/packages/schema/dist/routes.js.map +1 -0
- package/packages/schema/dist/schemas.d.ts +297 -0
- package/packages/schema/dist/schemas.js +243 -0
- package/packages/schema/dist/schemas.js.map +1 -0
- package/packages/schema/dist/textFiles.d.ts +5 -0
- package/packages/schema/dist/textFiles.js +66 -0
- package/packages/schema/dist/textFiles.js.map +1 -0
- package/packages/schema/package.json +26 -0
- package/packages/schema/src/ark.ts +29 -0
- package/packages/schema/src/index.ts +5 -0
- package/packages/schema/src/routes.ts +22 -0
- package/packages/schema/src/schemas.test.ts +123 -0
- package/packages/schema/src/schemas.ts +287 -0
- package/packages/schema/src/textFiles.test.ts +23 -0
- package/packages/schema/src/textFiles.ts +66 -0
- package/packages/schema/tsconfig.json +15 -0
- package/pilothub +46 -0
- package/playwright.config.ts +33 -0
- package/public/.well-known/pilothub.json +6 -0
- package/public/api/v1/openapi.json +379 -0
- package/public/favicon.ico +0 -0
- package/public/logo192.png +0 -0
- package/public/logo512.png +0 -0
- package/public/manifest.json +25 -0
- package/public/og.png +0 -0
- package/public/og.svg +98 -0
- package/public/pilot-logo.png +0 -0
- package/public/pilot-mark.png +0 -0
- package/public/robots.txt +3 -0
- package/public/tanstack-circle-logo.png +0 -0
- package/public/tanstack-word-logo-white.svg +1 -0
- package/scripts/check-peer-deps.ts +56 -0
- package/scripts/docs-list.ts +148 -0
- package/scripts/run-playwright-local.sh +14 -0
- package/server/og/fetchSkillOgMeta.ts +27 -0
- package/server/og/fetchSoulOgMeta.ts +27 -0
- package/server/og/ogAssets.ts +80 -0
- package/server/og/skillOgSvg.test.ts +59 -0
- package/server/og/skillOgSvg.ts +258 -0
- package/server/og/soulOgSvg.ts +209 -0
- package/server/routes/og/skill.png.ts +103 -0
- package/server/routes/og/soul.png.ts +111 -0
- package/src/__tests__/skill-detail-page.test.tsx +86 -0
- package/src/__tests__/skills-index.test.tsx +145 -0
- package/src/__tests__/upload.route.test.tsx +228 -0
- package/src/components/AppProviders.tsx +19 -0
- package/src/components/ClientOnly.tsx +18 -0
- package/src/components/Footer.tsx +29 -0
- package/src/components/Header.tsx +295 -0
- package/src/components/InstallSwitcher.tsx +53 -0
- package/src/components/SkillCard.tsx +36 -0
- package/src/components/SkillDetailPage.tsx +817 -0
- package/src/components/SkillDiffCard.tsx +485 -0
- package/src/components/SoulCard.tsx +19 -0
- package/src/components/SoulDetailPage.tsx +263 -0
- package/src/components/UserBootstrap.tsx +18 -0
- package/src/components/ui/dropdown-menu.tsx +67 -0
- package/src/components/ui/toggle-group.tsx +35 -0
- package/src/convex/client.ts +3 -0
- package/src/lib/badges.ts +29 -0
- package/src/lib/diffing.test.ts +163 -0
- package/src/lib/diffing.ts +106 -0
- package/src/lib/gravatar.test.ts +9 -0
- package/src/lib/gravatar.ts +158 -0
- package/src/lib/og.test.ts +142 -0
- package/src/lib/og.ts +156 -0
- package/src/lib/publicUser.ts +39 -0
- package/src/lib/roles.ts +19 -0
- package/src/lib/site.test.ts +130 -0
- package/src/lib/site.ts +84 -0
- package/src/lib/theme-transition.test.ts +134 -0
- package/src/lib/theme-transition.ts +134 -0
- package/src/lib/theme.test.tsx +88 -0
- package/src/lib/theme.ts +43 -0
- package/src/lib/uploadFiles.jsdom.test.ts +33 -0
- package/src/lib/uploadFiles.test.ts +123 -0
- package/src/lib/uploadFiles.ts +245 -0
- package/src/lib/uploadUtils.test.ts +78 -0
- package/src/lib/uploadUtils.ts +93 -0
- package/src/lib/useAuthStatus.ts +12 -0
- package/src/lib/utils.test.ts +9 -0
- package/src/lib/utils.ts +6 -0
- package/src/logo.svg +12 -0
- package/src/routeTree.gen.ts +345 -0
- package/src/router.tsx +17 -0
- package/src/routes/$owner/$slug.tsx +55 -0
- package/src/routes/__root.tsx +136 -0
- package/src/routes/admin.tsx +11 -0
- package/src/routes/cli/auth.tsx +168 -0
- package/src/routes/dashboard.tsx +97 -0
- package/src/routes/import.tsx +415 -0
- package/src/routes/index.tsx +252 -0
- package/src/routes/management.tsx +529 -0
- package/src/routes/settings.tsx +203 -0
- package/src/routes/skills/index.tsx +422 -0
- package/src/routes/souls/$slug.tsx +55 -0
- package/src/routes/souls/index.tsx +243 -0
- package/src/routes/stars.tsx +68 -0
- package/src/routes/u/$handle.tsx +307 -0
- package/src/routes/upload/utils.ts +81 -0
- package/src/routes/upload.tsx +499 -0
- package/src/styles.css +2718 -0
- package/tsconfig.json +24 -0
- package/tsconfig.oxlint.json +16 -0
- package/vercel.json +8 -0
- package/vite.config.ts +48 -0
- package/vitest.config.ts +47 -0
- package/vitest.e2e.config.ts +11 -0
- package/vitest.setup.ts +1 -0
|
@@ -0,0 +1,840 @@
|
|
|
1
|
+
import { ConvexError, v } from 'convex/values'
|
|
2
|
+
import { internal } from './_generated/api'
|
|
3
|
+
import type { Doc, Id } from './_generated/dataModel'
|
|
4
|
+
import type { ActionCtx } from './_generated/server'
|
|
5
|
+
import { action, internalAction, internalMutation, internalQuery } from './_generated/server'
|
|
6
|
+
import { assertRole, requireUserFromAction } from './lib/access'
|
|
7
|
+
import { buildSkillSummaryBackfillPatch, type ParsedSkillData } from './lib/skillBackfill'
|
|
8
|
+
import { hashSkillFiles } from './lib/skills'
|
|
9
|
+
|
|
10
|
+
const DEFAULT_BATCH_SIZE = 50
|
|
11
|
+
const MAX_BATCH_SIZE = 200
|
|
12
|
+
const DEFAULT_MAX_BATCHES = 20
|
|
13
|
+
const MAX_MAX_BATCHES = 200
|
|
14
|
+
|
|
15
|
+
type BackfillStats = {
|
|
16
|
+
skillsScanned: number
|
|
17
|
+
skillsPatched: number
|
|
18
|
+
versionsPatched: number
|
|
19
|
+
missingLatestVersion: number
|
|
20
|
+
missingReadme: number
|
|
21
|
+
missingStorageBlob: number
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
type BackfillPageItem =
|
|
25
|
+
| {
|
|
26
|
+
kind: 'ok'
|
|
27
|
+
skillId: Id<'skills'>
|
|
28
|
+
versionId: Id<'skillVersions'>
|
|
29
|
+
skillSummary: Doc<'skills'>['summary']
|
|
30
|
+
versionParsed: Doc<'skillVersions'>['parsed']
|
|
31
|
+
readmeStorageId: Id<'_storage'>
|
|
32
|
+
}
|
|
33
|
+
| { kind: 'missingLatestVersion'; skillId: Id<'skills'> }
|
|
34
|
+
| { kind: 'missingVersionDoc'; skillId: Id<'skills'>; versionId: Id<'skillVersions'> }
|
|
35
|
+
| { kind: 'missingReadme'; skillId: Id<'skills'>; versionId: Id<'skillVersions'> }
|
|
36
|
+
|
|
37
|
+
type BackfillPageResult = {
|
|
38
|
+
items: BackfillPageItem[]
|
|
39
|
+
cursor: string | null
|
|
40
|
+
isDone: boolean
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export const getSkillBackfillPageInternal = internalQuery({
|
|
44
|
+
args: {
|
|
45
|
+
cursor: v.optional(v.string()),
|
|
46
|
+
batchSize: v.optional(v.number()),
|
|
47
|
+
},
|
|
48
|
+
handler: async (ctx, args): Promise<BackfillPageResult> => {
|
|
49
|
+
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
|
|
50
|
+
const { page, isDone, continueCursor } = await ctx.db
|
|
51
|
+
.query('skills')
|
|
52
|
+
.order('asc')
|
|
53
|
+
.paginate({ cursor: args.cursor ?? null, numItems: batchSize })
|
|
54
|
+
|
|
55
|
+
const items: BackfillPageItem[] = []
|
|
56
|
+
for (const skill of page) {
|
|
57
|
+
if (!skill.latestVersionId) {
|
|
58
|
+
items.push({ kind: 'missingLatestVersion', skillId: skill._id })
|
|
59
|
+
continue
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const version = await ctx.db.get(skill.latestVersionId)
|
|
63
|
+
if (!version) {
|
|
64
|
+
items.push({
|
|
65
|
+
kind: 'missingVersionDoc',
|
|
66
|
+
skillId: skill._id,
|
|
67
|
+
versionId: skill.latestVersionId,
|
|
68
|
+
})
|
|
69
|
+
continue
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const readmeFile = version.files.find(
|
|
73
|
+
(file) => file.path.toLowerCase() === 'skill.md' || file.path.toLowerCase() === 'skills.md',
|
|
74
|
+
)
|
|
75
|
+
if (!readmeFile) {
|
|
76
|
+
items.push({ kind: 'missingReadme', skillId: skill._id, versionId: version._id })
|
|
77
|
+
continue
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
items.push({
|
|
81
|
+
kind: 'ok',
|
|
82
|
+
skillId: skill._id,
|
|
83
|
+
versionId: version._id,
|
|
84
|
+
skillSummary: skill.summary,
|
|
85
|
+
versionParsed: version.parsed,
|
|
86
|
+
readmeStorageId: readmeFile.storageId,
|
|
87
|
+
})
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
return { items, cursor: continueCursor, isDone }
|
|
91
|
+
},
|
|
92
|
+
})
|
|
93
|
+
|
|
94
|
+
export const applySkillBackfillPatchInternal = internalMutation({
|
|
95
|
+
args: {
|
|
96
|
+
skillId: v.id('skills'),
|
|
97
|
+
versionId: v.id('skillVersions'),
|
|
98
|
+
summary: v.optional(v.string()),
|
|
99
|
+
parsed: v.optional(
|
|
100
|
+
v.object({
|
|
101
|
+
frontmatter: v.record(v.string(), v.any()),
|
|
102
|
+
metadata: v.optional(v.any()),
|
|
103
|
+
pilotbot: v.optional(v.any()),
|
|
104
|
+
}),
|
|
105
|
+
),
|
|
106
|
+
},
|
|
107
|
+
handler: async (ctx, args) => {
|
|
108
|
+
const now = Date.now()
|
|
109
|
+
if (typeof args.summary === 'string') {
|
|
110
|
+
await ctx.db.patch(args.skillId, { summary: args.summary, updatedAt: now })
|
|
111
|
+
}
|
|
112
|
+
if (args.parsed) {
|
|
113
|
+
await ctx.db.patch(args.versionId, { parsed: args.parsed })
|
|
114
|
+
}
|
|
115
|
+
return { ok: true as const }
|
|
116
|
+
},
|
|
117
|
+
})
|
|
118
|
+
|
|
119
|
+
export type BackfillActionArgs = {
|
|
120
|
+
dryRun?: boolean
|
|
121
|
+
batchSize?: number
|
|
122
|
+
maxBatches?: number
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export type BackfillActionResult = { ok: true; stats: BackfillStats }
|
|
126
|
+
|
|
127
|
+
export async function backfillSkillSummariesInternalHandler(
|
|
128
|
+
ctx: ActionCtx,
|
|
129
|
+
args: BackfillActionArgs,
|
|
130
|
+
): Promise<BackfillActionResult> {
|
|
131
|
+
const dryRun = Boolean(args.dryRun)
|
|
132
|
+
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
|
|
133
|
+
const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES)
|
|
134
|
+
|
|
135
|
+
const totals: BackfillStats = {
|
|
136
|
+
skillsScanned: 0,
|
|
137
|
+
skillsPatched: 0,
|
|
138
|
+
versionsPatched: 0,
|
|
139
|
+
missingLatestVersion: 0,
|
|
140
|
+
missingReadme: 0,
|
|
141
|
+
missingStorageBlob: 0,
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
let cursor: string | null = null
|
|
145
|
+
let isDone = false
|
|
146
|
+
|
|
147
|
+
for (let i = 0; i < maxBatches; i++) {
|
|
148
|
+
const page = (await ctx.runQuery(internal.maintenance.getSkillBackfillPageInternal, {
|
|
149
|
+
cursor: cursor ?? undefined,
|
|
150
|
+
batchSize,
|
|
151
|
+
})) as BackfillPageResult
|
|
152
|
+
|
|
153
|
+
cursor = page.cursor
|
|
154
|
+
isDone = page.isDone
|
|
155
|
+
|
|
156
|
+
for (const item of page.items) {
|
|
157
|
+
totals.skillsScanned++
|
|
158
|
+
if (item.kind === 'missingLatestVersion') {
|
|
159
|
+
totals.missingLatestVersion++
|
|
160
|
+
continue
|
|
161
|
+
}
|
|
162
|
+
if (item.kind === 'missingVersionDoc') {
|
|
163
|
+
totals.missingLatestVersion++
|
|
164
|
+
continue
|
|
165
|
+
}
|
|
166
|
+
if (item.kind === 'missingReadme') {
|
|
167
|
+
totals.missingReadme++
|
|
168
|
+
continue
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
const blob = await ctx.storage.get(item.readmeStorageId)
|
|
172
|
+
if (!blob) {
|
|
173
|
+
totals.missingStorageBlob++
|
|
174
|
+
continue
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
const readmeText = await blob.text()
|
|
178
|
+
const patch = buildSkillSummaryBackfillPatch({
|
|
179
|
+
readmeText,
|
|
180
|
+
currentSummary: item.skillSummary ?? undefined,
|
|
181
|
+
currentParsed: item.versionParsed as ParsedSkillData,
|
|
182
|
+
})
|
|
183
|
+
|
|
184
|
+
if (!patch.summary && !patch.parsed) continue
|
|
185
|
+
if (patch.summary) totals.skillsPatched++
|
|
186
|
+
if (patch.parsed) totals.versionsPatched++
|
|
187
|
+
|
|
188
|
+
if (dryRun) continue
|
|
189
|
+
|
|
190
|
+
await ctx.runMutation(internal.maintenance.applySkillBackfillPatchInternal, {
|
|
191
|
+
skillId: item.skillId,
|
|
192
|
+
versionId: item.versionId,
|
|
193
|
+
summary: patch.summary,
|
|
194
|
+
parsed: patch.parsed,
|
|
195
|
+
})
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
if (isDone) break
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
if (!isDone) {
|
|
202
|
+
throw new ConvexError('Backfill incomplete (maxBatches reached)')
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
return { ok: true as const, stats: totals }
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
export const backfillSkillSummariesInternal = internalAction({
|
|
209
|
+
args: {
|
|
210
|
+
dryRun: v.optional(v.boolean()),
|
|
211
|
+
batchSize: v.optional(v.number()),
|
|
212
|
+
maxBatches: v.optional(v.number()),
|
|
213
|
+
},
|
|
214
|
+
handler: backfillSkillSummariesInternalHandler,
|
|
215
|
+
})
|
|
216
|
+
|
|
217
|
+
export const backfillSkillSummaries: ReturnType<typeof action> = action({
|
|
218
|
+
args: {
|
|
219
|
+
dryRun: v.optional(v.boolean()),
|
|
220
|
+
batchSize: v.optional(v.number()),
|
|
221
|
+
maxBatches: v.optional(v.number()),
|
|
222
|
+
},
|
|
223
|
+
handler: async (ctx, args): Promise<BackfillActionResult> => {
|
|
224
|
+
const { user } = await requireUserFromAction(ctx)
|
|
225
|
+
assertRole(user, ['admin'])
|
|
226
|
+
return ctx.runAction(
|
|
227
|
+
internal.maintenance.backfillSkillSummariesInternal,
|
|
228
|
+
args,
|
|
229
|
+
) as Promise<BackfillActionResult>
|
|
230
|
+
},
|
|
231
|
+
})
|
|
232
|
+
|
|
233
|
+
export const scheduleBackfillSkillSummaries: ReturnType<typeof action> = action({
|
|
234
|
+
args: { dryRun: v.optional(v.boolean()) },
|
|
235
|
+
handler: async (ctx, args) => {
|
|
236
|
+
const { user } = await requireUserFromAction(ctx)
|
|
237
|
+
assertRole(user, ['admin'])
|
|
238
|
+
await ctx.scheduler.runAfter(0, internal.maintenance.backfillSkillSummariesInternal, {
|
|
239
|
+
dryRun: Boolean(args.dryRun),
|
|
240
|
+
batchSize: DEFAULT_BATCH_SIZE,
|
|
241
|
+
maxBatches: DEFAULT_MAX_BATCHES,
|
|
242
|
+
})
|
|
243
|
+
return { ok: true as const }
|
|
244
|
+
},
|
|
245
|
+
})
|
|
246
|
+
|
|
247
|
+
type FingerprintBackfillStats = {
|
|
248
|
+
versionsScanned: number
|
|
249
|
+
versionsPatched: number
|
|
250
|
+
fingerprintsInserted: number
|
|
251
|
+
fingerprintMismatches: number
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
type FingerprintBackfillPageItem = {
|
|
255
|
+
skillId: Id<'skills'>
|
|
256
|
+
versionId: Id<'skillVersions'>
|
|
257
|
+
versionFingerprint?: string
|
|
258
|
+
files: Array<{ path: string; sha256: string }>
|
|
259
|
+
existingEntries: Array<{ id: Id<'skillVersionFingerprints'>; fingerprint: string }>
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
type FingerprintBackfillPageResult = {
|
|
263
|
+
items: FingerprintBackfillPageItem[]
|
|
264
|
+
cursor: string | null
|
|
265
|
+
isDone: boolean
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
type BadgeBackfillStats = {
|
|
269
|
+
skillsScanned: number
|
|
270
|
+
skillsPatched: number
|
|
271
|
+
highlightsPatched: number
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
type SkillBadgeTableBackfillStats = {
|
|
275
|
+
skillsScanned: number
|
|
276
|
+
recordsInserted: number
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
type BadgeBackfillPageItem = {
|
|
280
|
+
skillId: Id<'skills'>
|
|
281
|
+
ownerUserId: Id<'users'>
|
|
282
|
+
createdAt?: number
|
|
283
|
+
updatedAt?: number
|
|
284
|
+
batch?: string
|
|
285
|
+
badges?: Doc<'skills'>['badges']
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
type BadgeBackfillPageResult = {
|
|
289
|
+
items: BadgeBackfillPageItem[]
|
|
290
|
+
cursor: string | null
|
|
291
|
+
isDone: boolean
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
type BadgeKind = Doc<'skillBadges'>['kind']
|
|
295
|
+
|
|
296
|
+
export const getSkillFingerprintBackfillPageInternal = internalQuery({
|
|
297
|
+
args: {
|
|
298
|
+
cursor: v.optional(v.string()),
|
|
299
|
+
batchSize: v.optional(v.number()),
|
|
300
|
+
},
|
|
301
|
+
handler: async (ctx, args): Promise<FingerprintBackfillPageResult> => {
|
|
302
|
+
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
|
|
303
|
+
const { page, isDone, continueCursor } = await ctx.db
|
|
304
|
+
.query('skillVersions')
|
|
305
|
+
.order('asc')
|
|
306
|
+
.paginate({ cursor: args.cursor ?? null, numItems: batchSize })
|
|
307
|
+
|
|
308
|
+
const items: FingerprintBackfillPageItem[] = []
|
|
309
|
+
for (const version of page) {
|
|
310
|
+
const existingEntries = await ctx.db
|
|
311
|
+
.query('skillVersionFingerprints')
|
|
312
|
+
.withIndex('by_version', (q) => q.eq('versionId', version._id))
|
|
313
|
+
.take(20)
|
|
314
|
+
|
|
315
|
+
const normalizedFiles = version.files.map((file) => ({
|
|
316
|
+
path: file.path,
|
|
317
|
+
sha256: file.sha256,
|
|
318
|
+
}))
|
|
319
|
+
|
|
320
|
+
const hasAnyEntry = existingEntries.length > 0
|
|
321
|
+
const entryFingerprints = new Set(existingEntries.map((entry) => entry.fingerprint))
|
|
322
|
+
const hasFingerprintMismatch =
|
|
323
|
+
typeof version.fingerprint === 'string' &&
|
|
324
|
+
hasAnyEntry &&
|
|
325
|
+
(entryFingerprints.size !== 1 || !entryFingerprints.has(version.fingerprint))
|
|
326
|
+
const needsFingerprintField = !version.fingerprint
|
|
327
|
+
const needsFingerprintEntry = !hasAnyEntry
|
|
328
|
+
|
|
329
|
+
if (!needsFingerprintField && !needsFingerprintEntry && !hasFingerprintMismatch) continue
|
|
330
|
+
|
|
331
|
+
items.push({
|
|
332
|
+
skillId: version.skillId,
|
|
333
|
+
versionId: version._id,
|
|
334
|
+
versionFingerprint: version.fingerprint ?? undefined,
|
|
335
|
+
files: normalizedFiles,
|
|
336
|
+
existingEntries: existingEntries.map((entry) => ({
|
|
337
|
+
id: entry._id,
|
|
338
|
+
fingerprint: entry.fingerprint,
|
|
339
|
+
})),
|
|
340
|
+
})
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
return { items, cursor: continueCursor, isDone }
|
|
344
|
+
},
|
|
345
|
+
})
|
|
346
|
+
|
|
347
|
+
export const applySkillFingerprintBackfillPatchInternal = internalMutation({
|
|
348
|
+
args: {
|
|
349
|
+
versionId: v.id('skillVersions'),
|
|
350
|
+
fingerprint: v.string(),
|
|
351
|
+
patchVersion: v.boolean(),
|
|
352
|
+
replaceEntries: v.boolean(),
|
|
353
|
+
existingEntryIds: v.optional(v.array(v.id('skillVersionFingerprints'))),
|
|
354
|
+
},
|
|
355
|
+
handler: async (ctx, args) => {
|
|
356
|
+
const version = await ctx.db.get(args.versionId)
|
|
357
|
+
if (!version) return { ok: false as const, reason: 'missingVersion' as const }
|
|
358
|
+
|
|
359
|
+
const now = Date.now()
|
|
360
|
+
|
|
361
|
+
if (args.patchVersion) {
|
|
362
|
+
await ctx.db.patch(version._id, { fingerprint: args.fingerprint })
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
if (args.replaceEntries) {
|
|
366
|
+
const existing = args.existingEntryIds ?? []
|
|
367
|
+
for (const id of existing) {
|
|
368
|
+
await ctx.db.delete(id)
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
await ctx.db.insert('skillVersionFingerprints', {
|
|
372
|
+
skillId: version.skillId,
|
|
373
|
+
versionId: version._id,
|
|
374
|
+
fingerprint: args.fingerprint,
|
|
375
|
+
createdAt: now,
|
|
376
|
+
})
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
return { ok: true as const }
|
|
380
|
+
},
|
|
381
|
+
})
|
|
382
|
+
|
|
383
|
+
export type FingerprintBackfillActionArgs = {
|
|
384
|
+
dryRun?: boolean
|
|
385
|
+
batchSize?: number
|
|
386
|
+
maxBatches?: number
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
export type FingerprintBackfillActionResult = { ok: true; stats: FingerprintBackfillStats }
|
|
390
|
+
|
|
391
|
+
export async function backfillSkillFingerprintsInternalHandler(
|
|
392
|
+
ctx: ActionCtx,
|
|
393
|
+
args: FingerprintBackfillActionArgs,
|
|
394
|
+
): Promise<FingerprintBackfillActionResult> {
|
|
395
|
+
const dryRun = Boolean(args.dryRun)
|
|
396
|
+
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
|
|
397
|
+
const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES)
|
|
398
|
+
|
|
399
|
+
const totals: FingerprintBackfillStats = {
|
|
400
|
+
versionsScanned: 0,
|
|
401
|
+
versionsPatched: 0,
|
|
402
|
+
fingerprintsInserted: 0,
|
|
403
|
+
fingerprintMismatches: 0,
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
let cursor: string | null = null
|
|
407
|
+
let isDone = false
|
|
408
|
+
|
|
409
|
+
for (let i = 0; i < maxBatches; i++) {
|
|
410
|
+
const page = (await ctx.runQuery(internal.maintenance.getSkillFingerprintBackfillPageInternal, {
|
|
411
|
+
cursor: cursor ?? undefined,
|
|
412
|
+
batchSize,
|
|
413
|
+
})) as FingerprintBackfillPageResult
|
|
414
|
+
|
|
415
|
+
cursor = page.cursor
|
|
416
|
+
isDone = page.isDone
|
|
417
|
+
|
|
418
|
+
for (const item of page.items) {
|
|
419
|
+
totals.versionsScanned++
|
|
420
|
+
|
|
421
|
+
const fingerprint = await hashSkillFiles(item.files)
|
|
422
|
+
|
|
423
|
+
const existingFingerprints = new Set(item.existingEntries.map((entry) => entry.fingerprint))
|
|
424
|
+
const hasAnyEntry = item.existingEntries.length > 0
|
|
425
|
+
const entryIsCorrect =
|
|
426
|
+
hasAnyEntry && existingFingerprints.size === 1 && existingFingerprints.has(fingerprint)
|
|
427
|
+
const versionFingerprintIsCorrect = item.versionFingerprint === fingerprint
|
|
428
|
+
|
|
429
|
+
if (hasAnyEntry && !entryIsCorrect) totals.fingerprintMismatches++
|
|
430
|
+
|
|
431
|
+
const shouldPatchVersion = !versionFingerprintIsCorrect
|
|
432
|
+
const shouldReplaceEntries = !entryIsCorrect
|
|
433
|
+
if (!shouldPatchVersion && !shouldReplaceEntries) continue
|
|
434
|
+
|
|
435
|
+
if (shouldPatchVersion) totals.versionsPatched++
|
|
436
|
+
if (shouldReplaceEntries) totals.fingerprintsInserted++
|
|
437
|
+
|
|
438
|
+
if (dryRun) continue
|
|
439
|
+
|
|
440
|
+
await ctx.runMutation(internal.maintenance.applySkillFingerprintBackfillPatchInternal, {
|
|
441
|
+
versionId: item.versionId,
|
|
442
|
+
fingerprint,
|
|
443
|
+
patchVersion: shouldPatchVersion,
|
|
444
|
+
replaceEntries: shouldReplaceEntries,
|
|
445
|
+
existingEntryIds: shouldReplaceEntries ? item.existingEntries.map((entry) => entry.id) : [],
|
|
446
|
+
})
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
if (isDone) break
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
if (!isDone) {
|
|
453
|
+
throw new ConvexError('Backfill incomplete (maxBatches reached)')
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
return { ok: true as const, stats: totals }
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
export const backfillSkillFingerprintsInternal = internalAction({
|
|
460
|
+
args: {
|
|
461
|
+
dryRun: v.optional(v.boolean()),
|
|
462
|
+
batchSize: v.optional(v.number()),
|
|
463
|
+
maxBatches: v.optional(v.number()),
|
|
464
|
+
},
|
|
465
|
+
handler: backfillSkillFingerprintsInternalHandler,
|
|
466
|
+
})
|
|
467
|
+
|
|
468
|
+
export const backfillSkillFingerprints: ReturnType<typeof action> = action({
|
|
469
|
+
args: {
|
|
470
|
+
dryRun: v.optional(v.boolean()),
|
|
471
|
+
batchSize: v.optional(v.number()),
|
|
472
|
+
maxBatches: v.optional(v.number()),
|
|
473
|
+
},
|
|
474
|
+
handler: async (ctx, args): Promise<FingerprintBackfillActionResult> => {
|
|
475
|
+
const { user } = await requireUserFromAction(ctx)
|
|
476
|
+
assertRole(user, ['admin'])
|
|
477
|
+
return ctx.runAction(
|
|
478
|
+
internal.maintenance.backfillSkillFingerprintsInternal,
|
|
479
|
+
args,
|
|
480
|
+
) as Promise<FingerprintBackfillActionResult>
|
|
481
|
+
},
|
|
482
|
+
})
|
|
483
|
+
|
|
484
|
+
export const scheduleBackfillSkillFingerprints: ReturnType<typeof action> = action({
|
|
485
|
+
args: { dryRun: v.optional(v.boolean()) },
|
|
486
|
+
handler: async (ctx, args) => {
|
|
487
|
+
const { user } = await requireUserFromAction(ctx)
|
|
488
|
+
assertRole(user, ['admin'])
|
|
489
|
+
await ctx.scheduler.runAfter(0, internal.maintenance.backfillSkillFingerprintsInternal, {
|
|
490
|
+
dryRun: Boolean(args.dryRun),
|
|
491
|
+
batchSize: DEFAULT_BATCH_SIZE,
|
|
492
|
+
maxBatches: DEFAULT_MAX_BATCHES,
|
|
493
|
+
})
|
|
494
|
+
return { ok: true as const }
|
|
495
|
+
},
|
|
496
|
+
})
|
|
497
|
+
|
|
498
|
+
export const getSkillBadgeBackfillPageInternal = internalQuery({
|
|
499
|
+
args: {
|
|
500
|
+
cursor: v.optional(v.string()),
|
|
501
|
+
batchSize: v.optional(v.number()),
|
|
502
|
+
},
|
|
503
|
+
handler: async (ctx, args): Promise<BadgeBackfillPageResult> => {
|
|
504
|
+
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
|
|
505
|
+
const { page, isDone, continueCursor } = await ctx.db
|
|
506
|
+
.query('skills')
|
|
507
|
+
.order('asc')
|
|
508
|
+
.paginate({ cursor: args.cursor ?? null, numItems: batchSize })
|
|
509
|
+
|
|
510
|
+
const items: BadgeBackfillPageItem[] = page.map((skill) => ({
|
|
511
|
+
skillId: skill._id,
|
|
512
|
+
ownerUserId: skill.ownerUserId,
|
|
513
|
+
createdAt: skill.createdAt ?? undefined,
|
|
514
|
+
updatedAt: skill.updatedAt ?? undefined,
|
|
515
|
+
batch: skill.batch ?? undefined,
|
|
516
|
+
badges: skill.badges ?? undefined,
|
|
517
|
+
}))
|
|
518
|
+
|
|
519
|
+
return { items, cursor: continueCursor, isDone }
|
|
520
|
+
},
|
|
521
|
+
})
|
|
522
|
+
|
|
523
|
+
export const applySkillBadgeBackfillPatchInternal = internalMutation({
|
|
524
|
+
args: {
|
|
525
|
+
skillId: v.id('skills'),
|
|
526
|
+
badges: v.optional(
|
|
527
|
+
v.object({
|
|
528
|
+
redactionApproved: v.optional(
|
|
529
|
+
v.object({
|
|
530
|
+
byUserId: v.id('users'),
|
|
531
|
+
at: v.number(),
|
|
532
|
+
}),
|
|
533
|
+
),
|
|
534
|
+
highlighted: v.optional(
|
|
535
|
+
v.object({
|
|
536
|
+
byUserId: v.id('users'),
|
|
537
|
+
at: v.number(),
|
|
538
|
+
}),
|
|
539
|
+
),
|
|
540
|
+
official: v.optional(
|
|
541
|
+
v.object({
|
|
542
|
+
byUserId: v.id('users'),
|
|
543
|
+
at: v.number(),
|
|
544
|
+
}),
|
|
545
|
+
),
|
|
546
|
+
deprecated: v.optional(
|
|
547
|
+
v.object({
|
|
548
|
+
byUserId: v.id('users'),
|
|
549
|
+
at: v.number(),
|
|
550
|
+
}),
|
|
551
|
+
),
|
|
552
|
+
}),
|
|
553
|
+
),
|
|
554
|
+
},
|
|
555
|
+
handler: async (ctx, args) => {
|
|
556
|
+
await ctx.db.patch(args.skillId, { badges: args.badges ?? undefined, updatedAt: Date.now() })
|
|
557
|
+
return { ok: true as const }
|
|
558
|
+
},
|
|
559
|
+
})
|
|
560
|
+
|
|
561
|
+
export const upsertSkillBadgeRecordInternal = internalMutation({
|
|
562
|
+
args: {
|
|
563
|
+
skillId: v.id('skills'),
|
|
564
|
+
kind: v.union(
|
|
565
|
+
v.literal('highlighted'),
|
|
566
|
+
v.literal('official'),
|
|
567
|
+
v.literal('deprecated'),
|
|
568
|
+
v.literal('redactionApproved'),
|
|
569
|
+
),
|
|
570
|
+
byUserId: v.id('users'),
|
|
571
|
+
at: v.number(),
|
|
572
|
+
},
|
|
573
|
+
handler: async (ctx, args) => {
|
|
574
|
+
const existing = await ctx.db
|
|
575
|
+
.query('skillBadges')
|
|
576
|
+
.withIndex('by_skill_kind', (q) => q.eq('skillId', args.skillId).eq('kind', args.kind))
|
|
577
|
+
.unique()
|
|
578
|
+
if (existing) return { inserted: false as const }
|
|
579
|
+
await ctx.db.insert('skillBadges', {
|
|
580
|
+
skillId: args.skillId,
|
|
581
|
+
kind: args.kind,
|
|
582
|
+
byUserId: args.byUserId,
|
|
583
|
+
at: args.at,
|
|
584
|
+
})
|
|
585
|
+
return { inserted: true as const }
|
|
586
|
+
},
|
|
587
|
+
})
|
|
588
|
+
|
|
589
|
+
export type BadgeBackfillActionArgs = {
|
|
590
|
+
dryRun?: boolean
|
|
591
|
+
batchSize?: number
|
|
592
|
+
maxBatches?: number
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
export type BadgeBackfillActionResult = { ok: true; stats: BadgeBackfillStats }
|
|
596
|
+
|
|
597
|
+
export async function backfillSkillBadgesInternalHandler(
|
|
598
|
+
ctx: ActionCtx,
|
|
599
|
+
args: BadgeBackfillActionArgs,
|
|
600
|
+
): Promise<BadgeBackfillActionResult> {
|
|
601
|
+
const dryRun = Boolean(args.dryRun)
|
|
602
|
+
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
|
|
603
|
+
const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES)
|
|
604
|
+
|
|
605
|
+
const totals: BadgeBackfillStats = {
|
|
606
|
+
skillsScanned: 0,
|
|
607
|
+
skillsPatched: 0,
|
|
608
|
+
highlightsPatched: 0,
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
let cursor: string | null = null
|
|
612
|
+
let isDone = false
|
|
613
|
+
|
|
614
|
+
for (let i = 0; i < maxBatches; i++) {
|
|
615
|
+
const page = (await ctx.runQuery(internal.maintenance.getSkillBadgeBackfillPageInternal, {
|
|
616
|
+
cursor: cursor ?? undefined,
|
|
617
|
+
batchSize,
|
|
618
|
+
})) as BadgeBackfillPageResult
|
|
619
|
+
|
|
620
|
+
cursor = page.cursor
|
|
621
|
+
isDone = page.isDone
|
|
622
|
+
|
|
623
|
+
for (const item of page.items) {
|
|
624
|
+
totals.skillsScanned++
|
|
625
|
+
|
|
626
|
+
const shouldHighlight = item.batch === 'highlighted' && !item.badges?.highlighted
|
|
627
|
+
if (!shouldHighlight) continue
|
|
628
|
+
|
|
629
|
+
totals.skillsPatched++
|
|
630
|
+
totals.highlightsPatched++
|
|
631
|
+
|
|
632
|
+
if (dryRun) continue
|
|
633
|
+
|
|
634
|
+
const at = item.updatedAt ?? item.createdAt ?? Date.now()
|
|
635
|
+
await ctx.runMutation(internal.maintenance.applySkillBadgeBackfillPatchInternal, {
|
|
636
|
+
skillId: item.skillId,
|
|
637
|
+
badges: {
|
|
638
|
+
...item.badges,
|
|
639
|
+
highlighted: {
|
|
640
|
+
byUserId: item.ownerUserId,
|
|
641
|
+
at,
|
|
642
|
+
},
|
|
643
|
+
},
|
|
644
|
+
})
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
if (isDone) break
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
if (!isDone) {
|
|
651
|
+
throw new ConvexError('Backfill incomplete (maxBatches reached)')
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
return { ok: true as const, stats: totals }
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
export const backfillSkillBadgesInternal = internalAction({
|
|
658
|
+
args: {
|
|
659
|
+
dryRun: v.optional(v.boolean()),
|
|
660
|
+
batchSize: v.optional(v.number()),
|
|
661
|
+
maxBatches: v.optional(v.number()),
|
|
662
|
+
},
|
|
663
|
+
handler: backfillSkillBadgesInternalHandler,
|
|
664
|
+
})
|
|
665
|
+
|
|
666
|
+
export const backfillSkillBadges: ReturnType<typeof action> = action({
|
|
667
|
+
args: {
|
|
668
|
+
dryRun: v.optional(v.boolean()),
|
|
669
|
+
batchSize: v.optional(v.number()),
|
|
670
|
+
maxBatches: v.optional(v.number()),
|
|
671
|
+
},
|
|
672
|
+
handler: async (ctx, args): Promise<BadgeBackfillActionResult> => {
|
|
673
|
+
const { user } = await requireUserFromAction(ctx)
|
|
674
|
+
assertRole(user, ['admin'])
|
|
675
|
+
return ctx.runAction(
|
|
676
|
+
internal.maintenance.backfillSkillBadgesInternal,
|
|
677
|
+
args,
|
|
678
|
+
) as Promise<BadgeBackfillActionResult>
|
|
679
|
+
},
|
|
680
|
+
})
|
|
681
|
+
|
|
682
|
+
export const scheduleBackfillSkillBadges: ReturnType<typeof action> = action({
|
|
683
|
+
args: { dryRun: v.optional(v.boolean()) },
|
|
684
|
+
handler: async (ctx, args) => {
|
|
685
|
+
const { user } = await requireUserFromAction(ctx)
|
|
686
|
+
assertRole(user, ['admin'])
|
|
687
|
+
await ctx.scheduler.runAfter(0, internal.maintenance.backfillSkillBadgesInternal, {
|
|
688
|
+
dryRun: Boolean(args.dryRun),
|
|
689
|
+
batchSize: DEFAULT_BATCH_SIZE,
|
|
690
|
+
maxBatches: DEFAULT_MAX_BATCHES,
|
|
691
|
+
})
|
|
692
|
+
return { ok: true as const }
|
|
693
|
+
},
|
|
694
|
+
})
|
|
695
|
+
|
|
696
|
+
export type SkillBadgeTableBackfillActionResult = {
|
|
697
|
+
ok: true
|
|
698
|
+
stats: SkillBadgeTableBackfillStats
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
export async function backfillSkillBadgeTableInternalHandler(
|
|
702
|
+
ctx: ActionCtx,
|
|
703
|
+
args: BadgeBackfillActionArgs,
|
|
704
|
+
): Promise<SkillBadgeTableBackfillActionResult> {
|
|
705
|
+
const dryRun = Boolean(args.dryRun)
|
|
706
|
+
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE)
|
|
707
|
+
const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES)
|
|
708
|
+
|
|
709
|
+
const totals: SkillBadgeTableBackfillStats = {
|
|
710
|
+
skillsScanned: 0,
|
|
711
|
+
recordsInserted: 0,
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
let cursor: string | null = null
|
|
715
|
+
let isDone = false
|
|
716
|
+
|
|
717
|
+
for (let i = 0; i < maxBatches; i++) {
|
|
718
|
+
const page = (await ctx.runQuery(internal.maintenance.getSkillBadgeBackfillPageInternal, {
|
|
719
|
+
cursor: cursor ?? undefined,
|
|
720
|
+
batchSize,
|
|
721
|
+
})) as BadgeBackfillPageResult
|
|
722
|
+
|
|
723
|
+
cursor = page.cursor
|
|
724
|
+
isDone = page.isDone
|
|
725
|
+
|
|
726
|
+
for (const item of page.items) {
|
|
727
|
+
totals.skillsScanned++
|
|
728
|
+
const badges = item.badges ?? {}
|
|
729
|
+
const entries: Array<{ kind: BadgeKind; byUserId: Id<'users'>; at: number }> = []
|
|
730
|
+
|
|
731
|
+
if (badges.redactionApproved) {
|
|
732
|
+
entries.push({
|
|
733
|
+
kind: 'redactionApproved',
|
|
734
|
+
byUserId: badges.redactionApproved.byUserId,
|
|
735
|
+
at: badges.redactionApproved.at,
|
|
736
|
+
})
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
if (badges.official) {
|
|
740
|
+
entries.push({
|
|
741
|
+
kind: 'official',
|
|
742
|
+
byUserId: badges.official.byUserId,
|
|
743
|
+
at: badges.official.at,
|
|
744
|
+
})
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
if (badges.deprecated) {
|
|
748
|
+
entries.push({
|
|
749
|
+
kind: 'deprecated',
|
|
750
|
+
byUserId: badges.deprecated.byUserId,
|
|
751
|
+
at: badges.deprecated.at,
|
|
752
|
+
})
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
const highlighted =
|
|
756
|
+
badges.highlighted ??
|
|
757
|
+
(item.batch === 'highlighted'
|
|
758
|
+
? {
|
|
759
|
+
byUserId: item.ownerUserId,
|
|
760
|
+
at: item.updatedAt ?? item.createdAt ?? Date.now(),
|
|
761
|
+
}
|
|
762
|
+
: undefined)
|
|
763
|
+
|
|
764
|
+
if (highlighted) {
|
|
765
|
+
entries.push({
|
|
766
|
+
kind: 'highlighted',
|
|
767
|
+
byUserId: highlighted.byUserId,
|
|
768
|
+
at: highlighted.at,
|
|
769
|
+
})
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
if (dryRun) continue
|
|
773
|
+
|
|
774
|
+
for (const entry of entries) {
|
|
775
|
+
const result = await ctx.runMutation(internal.maintenance.upsertSkillBadgeRecordInternal, {
|
|
776
|
+
skillId: item.skillId,
|
|
777
|
+
kind: entry.kind,
|
|
778
|
+
byUserId: entry.byUserId,
|
|
779
|
+
at: entry.at,
|
|
780
|
+
})
|
|
781
|
+
if (result.inserted) {
|
|
782
|
+
totals.recordsInserted++
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
if (isDone) break
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
if (!isDone) {
|
|
791
|
+
throw new ConvexError('Backfill incomplete (maxBatches reached)')
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
return { ok: true as const, stats: totals }
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
export const backfillSkillBadgeTableInternal = internalAction({
|
|
798
|
+
args: {
|
|
799
|
+
dryRun: v.optional(v.boolean()),
|
|
800
|
+
batchSize: v.optional(v.number()),
|
|
801
|
+
maxBatches: v.optional(v.number()),
|
|
802
|
+
},
|
|
803
|
+
handler: backfillSkillBadgeTableInternalHandler,
|
|
804
|
+
})
|
|
805
|
+
|
|
806
|
+
export const backfillSkillBadgeTable: ReturnType<typeof action> = action({
|
|
807
|
+
args: {
|
|
808
|
+
dryRun: v.optional(v.boolean()),
|
|
809
|
+
batchSize: v.optional(v.number()),
|
|
810
|
+
maxBatches: v.optional(v.number()),
|
|
811
|
+
},
|
|
812
|
+
handler: async (ctx, args): Promise<SkillBadgeTableBackfillActionResult> => {
|
|
813
|
+
const { user } = await requireUserFromAction(ctx)
|
|
814
|
+
assertRole(user, ['admin'])
|
|
815
|
+
return ctx.runAction(
|
|
816
|
+
internal.maintenance.backfillSkillBadgeTableInternal,
|
|
817
|
+
args,
|
|
818
|
+
) as Promise<SkillBadgeTableBackfillActionResult>
|
|
819
|
+
},
|
|
820
|
+
})
|
|
821
|
+
|
|
822
|
+
export const scheduleBackfillSkillBadgeTable: ReturnType<typeof action> = action({
|
|
823
|
+
args: { dryRun: v.optional(v.boolean()) },
|
|
824
|
+
handler: async (ctx, args) => {
|
|
825
|
+
const { user } = await requireUserFromAction(ctx)
|
|
826
|
+
assertRole(user, ['admin'])
|
|
827
|
+
await ctx.scheduler.runAfter(0, internal.maintenance.backfillSkillBadgeTableInternal, {
|
|
828
|
+
dryRun: Boolean(args.dryRun),
|
|
829
|
+
batchSize: DEFAULT_BATCH_SIZE,
|
|
830
|
+
maxBatches: DEFAULT_MAX_BATCHES,
|
|
831
|
+
})
|
|
832
|
+
return { ok: true as const }
|
|
833
|
+
},
|
|
834
|
+
})
|
|
835
|
+
|
|
836
|
+
function clampInt(value: number, min: number, max: number) {
|
|
837
|
+
const rounded = Math.trunc(value)
|
|
838
|
+
if (!Number.isFinite(rounded)) return min
|
|
839
|
+
return Math.min(max, Math.max(min, rounded))
|
|
840
|
+
}
|