productfoo 0.0.0-stage → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +338 -2
- package/bin/foo +3 -0
- package/bin/foo.mjs +42 -0
- package/commands/bug.md +31 -0
- package/commands/discovery.md +28 -0
- package/commands/ideate.md +27 -0
- package/commands/init.md +28 -0
- package/commands/pitch.md +27 -0
- package/commands/preview.md +25 -0
- package/commands/research.md +31 -0
- package/commands/roast.md +27 -0
- package/commands/spec.md +30 -0
- package/config/pricing.yaml +198 -0
- package/config/routing-policy.yaml +171 -0
- package/dist-public/src/activeModel.d.ts +83 -0
- package/dist-public/src/activeModel.js +331 -0
- package/dist-public/src/agentAdapters.d.ts +122 -0
- package/dist-public/src/agentAdapters.js +178 -0
- package/dist-public/src/agentState.d.ts +43 -0
- package/dist-public/src/agentState.js +177 -0
- package/dist-public/src/bugCapsule.d.ts +94 -0
- package/dist-public/src/bugCapsule.js +227 -0
- package/dist-public/src/bugCapture.d.ts +26 -0
- package/dist-public/src/bugCapture.js +251 -0
- package/dist-public/src/bugOwnerLog.d.ts +93 -0
- package/dist-public/src/bugOwnerLog.js +304 -0
- package/dist-public/src/bugReportStore.d.ts +48 -0
- package/dist-public/src/bugReportStore.js +860 -0
- package/dist-public/src/bugSpecContext.d.ts +7 -0
- package/dist-public/src/bugSpecContext.js +129 -0
- package/dist-public/src/channelFreshness.d.ts +25 -0
- package/dist-public/src/channelFreshness.js +194 -0
- package/dist-public/src/cli.d.ts +30 -0
- package/dist-public/src/cli.js +1667 -0
- package/dist-public/src/cliArgs.d.ts +7 -0
- package/dist-public/src/cliArgs.js +44 -0
- package/dist-public/src/codeCatalog.d.ts +41 -0
- package/dist-public/src/codeCatalog.js +197 -0
- package/dist-public/src/commandContract.d.ts +84 -0
- package/dist-public/src/commandContract.js +390 -0
- package/dist-public/src/commandUsage.d.ts +3 -0
- package/dist-public/src/commandUsage.js +30 -0
- package/dist-public/src/commands/agent.d.ts +60 -0
- package/dist-public/src/commands/agent.js +1353 -0
- package/dist-public/src/commands/asp.d.ts +8 -0
- package/dist-public/src/commands/asp.js +669 -0
- package/dist-public/src/commands/evalCmd.d.ts +4 -0
- package/dist-public/src/commands/evalCmd.js +405 -0
- package/dist-public/src/commands/login.d.ts +25 -0
- package/dist-public/src/commands/login.js +94 -0
- package/dist-public/src/commands/loop.d.ts +16 -0
- package/dist-public/src/commands/loop.js +619 -0
- package/dist-public/src/commands/reconcile.d.ts +41 -0
- package/dist-public/src/commands/reconcile.js +292 -0
- package/dist-public/src/commands/runtime.d.ts +1 -0
- package/dist-public/src/commands/runtime.js +36 -0
- package/dist-public/src/commands/scanRedact.d.ts +8 -0
- package/dist-public/src/commands/scanRedact.js +266 -0
- package/dist-public/src/commands/setup.d.ts +28 -0
- package/dist-public/src/commands/setup.js +398 -0
- package/dist-public/src/commands/shared.d.ts +17 -0
- package/dist-public/src/commands/shared.js +147 -0
- package/dist-public/src/commands/source.d.ts +1 -0
- package/dist-public/src/commands/source.js +134 -0
- package/dist-public/src/commands/specCmd.d.ts +24 -0
- package/dist-public/src/commands/specCmd.js +794 -0
- package/dist-public/src/commands/update.d.ts +61 -0
- package/dist-public/src/commands/update.js +244 -0
- package/dist-public/src/comprehensionSignal.d.ts +20 -0
- package/dist-public/src/comprehensionSignal.js +166 -0
- package/dist-public/src/conductor/diversity.d.ts +48 -0
- package/dist-public/src/conductor/diversity.js +423 -0
- package/dist-public/src/conductor/engine.d.ts +7 -0
- package/dist-public/src/conductor/engine.js +295 -0
- package/dist-public/src/conductor/families/ideate.d.ts +2 -0
- package/dist-public/src/conductor/families/ideate.js +255 -0
- package/dist-public/src/conductor/families/interview.d.ts +2 -0
- package/dist-public/src/conductor/families/interview.js +39 -0
- package/dist-public/src/conductor/families/specSwarm.d.ts +2 -0
- package/dist-public/src/conductor/families/specSwarm.js +146 -0
- package/dist-public/src/conductor/familyDelivery.d.ts +6 -0
- package/dist-public/src/conductor/familyDelivery.js +37 -0
- package/dist-public/src/conductor/familyPackArtifact.d.ts +4 -0
- package/dist-public/src/conductor/familyPackArtifact.js +260 -0
- package/dist-public/src/conductor/ideateDecision.d.ts +6 -0
- package/dist-public/src/conductor/ideateDecision.js +237 -0
- package/dist-public/src/conductor/pack.d.ts +26 -0
- package/dist-public/src/conductor/pack.js +75 -0
- package/dist-public/src/conductor/types.d.ts +111 -0
- package/dist-public/src/conductor/types.js +7 -0
- package/dist-public/src/configPath.d.ts +2 -0
- package/dist-public/src/configPath.js +7 -0
- package/dist-public/src/discoveryArtifacts.d.ts +54 -0
- package/dist-public/src/discoveryArtifacts.js +467 -0
- package/dist-public/src/discoveryCommand.d.ts +127 -0
- package/dist-public/src/discoveryCommand.js +2148 -0
- package/dist-public/src/discoveryConnections.d.ts +339 -0
- package/dist-public/src/discoveryConnections.js +613 -0
- package/dist-public/src/discoveryDocs.d.ts +129 -0
- package/dist-public/src/discoveryDocs.js +577 -0
- package/dist-public/src/discoveryEvidence.d.ts +59 -0
- package/dist-public/src/discoveryEvidence.js +198 -0
- package/dist-public/src/discoveryFireflies.d.ts +25 -0
- package/dist-public/src/discoveryFireflies.js +551 -0
- package/dist-public/src/discoveryGmail.d.ts +12 -0
- package/dist-public/src/discoveryGmail.js +466 -0
- package/dist-public/src/discoveryGong.d.ts +12 -0
- package/dist-public/src/discoveryGong.js +491 -0
- package/dist-public/src/discoveryLearningRouting.d.ts +90 -0
- package/dist-public/src/discoveryLearningRouting.js +405 -0
- package/dist-public/src/discoveryMatching.d.ts +26 -0
- package/dist-public/src/discoveryMatching.js +73 -0
- package/dist-public/src/discoveryOutlook.d.ts +12 -0
- package/dist-public/src/discoveryOutlook.js +492 -0
- package/dist-public/src/discoveryProcess.d.ts +221 -0
- package/dist-public/src/discoveryProcess.js +703 -0
- package/dist-public/src/discoveryProof.d.ts +143 -0
- package/dist-public/src/discoveryProof.js +529 -0
- package/dist-public/src/discoveryProviders.d.ts +152 -0
- package/dist-public/src/discoveryProviders.js +756 -0
- package/dist-public/src/discoveryReadiness.d.ts +219 -0
- package/dist-public/src/discoveryReadiness.js +672 -0
- package/dist-public/src/discoveryReconcile.d.ts +57 -0
- package/dist-public/src/discoveryReconcile.js +424 -0
- package/dist-public/src/discoverySlack.d.ts +12 -0
- package/dist-public/src/discoverySlack.js +452 -0
- package/dist-public/src/discoverySources.d.ts +56 -0
- package/dist-public/src/discoverySources.js +322 -0
- package/dist-public/src/discoverySync.d.ts +52 -0
- package/dist-public/src/discoverySync.js +151 -0
- package/dist-public/src/discoverySynthesis.d.ts +57 -0
- package/dist-public/src/discoverySynthesis.js +215 -0
- package/dist-public/src/discoveryTeams.d.ts +12 -0
- package/dist-public/src/discoveryTeams.js +516 -0
- package/dist-public/src/discoveryZoom.d.ts +12 -0
- package/dist-public/src/discoveryZoom.js +523 -0
- package/dist-public/src/elicitationSurface.d.ts +13 -0
- package/dist-public/src/elicitationSurface.js +77 -0
- package/dist-public/src/errorReport.d.ts +99 -0
- package/dist-public/src/errorReport.js +443 -0
- package/dist-public/src/errorTaxonomy.d.ts +84 -0
- package/dist-public/src/errorTaxonomy.js +189 -0
- package/dist-public/src/evalCalibration.d.ts +82 -0
- package/dist-public/src/evalCalibration.js +169 -0
- package/dist-public/src/evalChecks.d.ts +21 -0
- package/dist-public/src/evalChecks.js +295 -0
- package/dist-public/src/evalCompile.d.ts +56 -0
- package/dist-public/src/evalCompile.js +58 -0
- package/dist-public/src/evalImprove.d.ts +60 -0
- package/dist-public/src/evalImprove.js +175 -0
- package/dist-public/src/evalJudge.d.ts +47 -0
- package/dist-public/src/evalJudge.js +143 -0
- package/dist-public/src/evalLabels.d.ts +100 -0
- package/dist-public/src/evalLabels.js +287 -0
- package/dist-public/src/evalRecord.d.ts +68 -0
- package/dist-public/src/evalRecord.js +165 -0
- package/dist-public/src/evalRoast.d.ts +88 -0
- package/dist-public/src/evalRoast.js +249 -0
- package/dist-public/src/evidenceIntake.d.ts +66 -0
- package/dist-public/src/evidenceIntake.js +356 -0
- package/dist-public/src/feedback.d.ts +78 -0
- package/dist-public/src/feedback.js +294 -0
- package/dist-public/src/firefliesClient.d.ts +106 -0
- package/dist-public/src/firefliesClient.js +258 -0
- package/dist-public/src/firstMoveTensions.d.ts +10 -0
- package/dist-public/src/firstMoveTensions.js +93 -0
- package/dist-public/src/gitRun.d.ts +21 -0
- package/dist-public/src/gitRun.js +50 -0
- package/dist-public/src/githubReconcile.d.ts +94 -0
- package/dist-public/src/githubReconcile.js +205 -0
- package/dist-public/src/gmailNango.d.ts +131 -0
- package/dist-public/src/gmailNango.js +329 -0
- package/dist-public/src/gongNango.d.ts +118 -0
- package/dist-public/src/gongNango.js +308 -0
- package/dist-public/src/grammarGate.d.ts +13 -0
- package/dist-public/src/grammarGate.js +72 -0
- package/dist-public/src/harnessGates.d.ts +65 -0
- package/dist-public/src/harnessGates.js +168 -0
- package/dist-public/src/humanSurfaceEval.d.ts +19 -0
- package/dist-public/src/humanSurfaceEval.js +156 -0
- package/dist-public/src/ideate.d.ts +43 -0
- package/dist-public/src/ideate.js +255 -0
- package/dist-public/src/ideateContext.d.ts +11 -0
- package/dist-public/src/ideateContext.js +83 -0
- package/dist-public/src/index.d.ts +2 -0
- package/dist-public/src/index.js +4 -0
- package/dist-public/src/initCommand.d.ts +25 -0
- package/dist-public/src/initCommand.js +558 -0
- package/dist-public/src/interviewGate.d.ts +25 -0
- package/dist-public/src/interviewGate.js +126 -0
- package/dist-public/src/jsonFile.d.ts +2 -0
- package/dist-public/src/jsonFile.js +22 -0
- package/dist-public/src/loopBudget.d.ts +182 -0
- package/dist-public/src/loopBudget.js +494 -0
- package/dist-public/src/loopContracts.d.ts +35 -0
- package/dist-public/src/loopContracts.js +364 -0
- package/dist-public/src/machineState.d.ts +1 -0
- package/dist-public/src/machineState.js +9 -0
- package/dist-public/src/notionNango.d.ts +93 -0
- package/dist-public/src/notionNango.js +279 -0
- package/dist-public/src/orchestrationResult.d.ts +11 -0
- package/dist-public/src/orchestrationResult.js +73 -0
- package/dist-public/src/outlookNango.d.ts +124 -0
- package/dist-public/src/outlookNango.js +327 -0
- package/dist-public/src/packetCorrections.d.ts +40 -0
- package/dist-public/src/packetCorrections.js +172 -0
- package/dist-public/src/packetGlossary.d.ts +6 -0
- package/dist-public/src/packetGlossary.js +42 -0
- package/dist-public/src/packetIndex.d.ts +54 -0
- package/dist-public/src/packetIndex.js +744 -0
- package/dist-public/src/packetIndexEntry.d.ts +3 -0
- package/dist-public/src/packetIndexEntry.js +128 -0
- package/dist-public/src/packetIndexServe.d.ts +13 -0
- package/dist-public/src/packetIndexServe.js +113 -0
- package/dist-public/src/packetIndexWorker.d.ts +7 -0
- package/dist-public/src/packetIndexWorker.js +45 -0
- package/dist-public/src/packetLifecycle.d.ts +30 -0
- package/dist-public/src/packetLifecycle.js +127 -0
- package/dist-public/src/packetReconcile.d.ts +73 -0
- package/dist-public/src/packetReconcile.js +386 -0
- package/dist-public/src/packetStatus.d.ts +125 -0
- package/dist-public/src/packetStatus.js +770 -0
- package/dist-public/src/packetValidation.d.ts +17 -0
- package/dist-public/src/packetValidation.js +333 -0
- package/dist-public/src/packetVerify.d.ts +73 -0
- package/dist-public/src/packetVerify.js +303 -0
- package/dist-public/src/packetWrite.d.ts +52 -0
- package/dist-public/src/packetWrite.js +448 -0
- package/dist-public/src/pitch.d.ts +38 -0
- package/dist-public/src/pitch.js +154 -0
- package/dist-public/src/processMemo.d.ts +8 -0
- package/dist-public/src/processMemo.js +23 -0
- package/dist-public/src/productDetect.d.ts +30 -0
- package/dist-public/src/productDetect.js +376 -0
- package/dist-public/src/profile.d.ts +62 -0
- package/dist-public/src/profile.js +334 -0
- package/dist-public/src/questionModel.d.ts +46 -0
- package/dist-public/src/questionModel.js +74 -0
- package/dist-public/src/questionText.d.ts +7 -0
- package/dist-public/src/questionText.js +38 -0
- package/dist-public/src/redactConfig.d.ts +2 -0
- package/dist-public/src/redactConfig.js +11 -0
- package/dist-public/src/redactEngine.d.ts +59 -0
- package/dist-public/src/redactEngine.js +509 -0
- package/dist-public/src/redactGate.d.ts +63 -0
- package/dist-public/src/redactGate.js +209 -0
- package/dist-public/src/repoRead.d.ts +29 -0
- package/dist-public/src/repoRead.js +160 -0
- package/dist-public/src/research.d.ts +242 -0
- package/dist-public/src/research.js +1314 -0
- package/dist-public/src/researchCompletion.d.ts +34 -0
- package/dist-public/src/researchCompletion.js +123 -0
- package/dist-public/src/researchExecution.d.ts +47 -0
- package/dist-public/src/researchExecution.js +309 -0
- package/dist-public/src/researchFormat.d.ts +21 -0
- package/dist-public/src/researchFormat.js +77 -0
- package/dist-public/src/researchGate.d.ts +86 -0
- package/dist-public/src/researchGate.js +1036 -0
- package/dist-public/src/researchReceipt.d.ts +76 -0
- package/dist-public/src/researchReceipt.js +148 -0
- package/dist-public/src/researchRender.d.ts +2 -0
- package/dist-public/src/researchRender.js +267 -0
- package/dist-public/src/researchSigningKey.d.ts +19 -0
- package/dist-public/src/researchSigningKey.js +95 -0
- package/dist-public/src/riskClassify.d.ts +42 -0
- package/dist-public/src/riskClassify.js +149 -0
- package/dist-public/src/routines.d.ts +42 -0
- package/dist-public/src/routines.js +269 -0
- package/dist-public/src/routingPolicy.d.ts +172 -0
- package/dist-public/src/routingPolicy.js +646 -0
- package/dist-public/src/secretInput.d.ts +8 -0
- package/dist-public/src/secretInput.js +89 -0
- package/dist-public/src/selfCheck.d.ts +7 -0
- package/dist-public/src/selfCheck.js +473 -0
- package/dist-public/src/selfCheckCache.d.ts +29 -0
- package/dist-public/src/selfCheckCache.js +147 -0
- package/dist-public/src/severityClass.d.ts +15 -0
- package/dist-public/src/severityClass.js +22 -0
- package/dist-public/src/skillClientPolicy.d.ts +8 -0
- package/dist-public/src/skillClientPolicy.js +16 -0
- package/dist-public/src/skillLocal.d.ts +98 -0
- package/dist-public/src/skillLocal.js +463 -0
- package/dist-public/src/skillTreeDigest.d.ts +9 -0
- package/dist-public/src/skillTreeDigest.js +64 -0
- package/dist-public/src/slackNango.d.ts +62 -0
- package/dist-public/src/slackNango.js +208 -0
- package/dist-public/src/social/arxiv.d.ts +19 -0
- package/dist-public/src/social/arxiv.js +96 -0
- package/dist-public/src/social/github.d.ts +2 -0
- package/dist-public/src/social/github.js +78 -0
- package/dist-public/src/social/hackernews.d.ts +2 -0
- package/dist-public/src/social/hackernews.js +67 -0
- package/dist-public/src/social/index.d.ts +10 -0
- package/dist-public/src/social/index.js +79 -0
- package/dist-public/src/social/reddit.d.ts +2 -0
- package/dist-public/src/social/reddit.js +125 -0
- package/dist-public/src/social/types.d.ts +73 -0
- package/dist-public/src/social/types.js +61 -0
- package/dist-public/src/social/xml.d.ts +10 -0
- package/dist-public/src/social/xml.js +49 -0
- package/dist-public/src/social/youtube.d.ts +7 -0
- package/dist-public/src/social/youtube.js +217 -0
- package/dist-public/src/sourceIngress.d.ts +488 -0
- package/dist-public/src/sourceIngress.js +1651 -0
- package/dist-public/src/specAfter.d.ts +187 -0
- package/dist-public/src/specAfter.js +814 -0
- package/dist-public/src/specBudget.d.ts +124 -0
- package/dist-public/src/specBudget.js +424 -0
- package/dist-public/src/specDecisionSignals.d.ts +38 -0
- package/dist-public/src/specDecisionSignals.js +121 -0
- package/dist-public/src/specLinkedCommits.d.ts +7 -0
- package/dist-public/src/specLinkedCommits.js +107 -0
- package/dist-public/src/specSections.d.ts +105 -0
- package/dist-public/src/specSections.js +718 -0
- package/dist-public/src/statusGlyphs.d.ts +18 -0
- package/dist-public/src/statusGlyphs.js +75 -0
- package/dist-public/src/swarmCommand.d.ts +10 -0
- package/dist-public/src/swarmCommand.js +96 -0
- package/dist-public/src/teamsNango.d.ts +114 -0
- package/dist-public/src/teamsNango.js +301 -0
- package/dist-public/src/tokenStore.d.ts +10 -0
- package/dist-public/src/tokenStore.js +77 -0
- package/dist-public/src/trustEnvelope.d.ts +78 -0
- package/dist-public/src/trustEnvelope.js +392 -0
- package/dist-public/src/untrustedText.d.ts +6 -0
- package/dist-public/src/untrustedText.js +20 -0
- package/dist-public/src/updateNotice.d.ts +41 -0
- package/dist-public/src/updateNotice.js +290 -0
- package/dist-public/src/vendorYaml.d.ts +1 -0
- package/dist-public/src/vendorYaml.js +7374 -0
- package/dist-public/src/workflowRunActuals.d.ts +67 -0
- package/dist-public/src/workflowRunActuals.js +184 -0
- package/dist-public/src/zendeskDiscovery.d.ts +12 -0
- package/dist-public/src/zendeskDiscovery.js +513 -0
- package/dist-public/src/zendeskNango.d.ts +131 -0
- package/dist-public/src/zendeskNango.js +344 -0
- package/dist-public/src/zoomNango.d.ts +106 -0
- package/dist-public/src/zoomNango.js +351 -0
- package/docs/CHANGELOG.md +32 -0
- package/docs/THIRD_PARTY_NOTICES.md +36 -0
- package/package.json +71 -4
- package/schemas/bug-capsule-receipt.schema.json +54 -0
- package/schemas/bug-capsule.schema.json +189 -0
- package/schemas/discovery-evidence-envelope.schema.json +79 -0
- package/schemas/discovery-processing-receipt.schema.json +71 -0
- package/schemas/discovery-service-contract.schema.json +189 -0
- package/skills/AGENTS.md +32 -0
- package/skills/README.md +287 -0
- package/skills/bug/SKILL.md +49 -0
- package/skills/build-preview-spec/SKILL.md +103 -0
- package/skills/capture-external-artifact/SKILL.md +23 -0
- package/skills/capture-external-artifact/agents/openai.yaml +4 -0
- package/skills/capture-external-artifact/references/extractor-libraries.md +27 -0
- package/skills/check-spec-health/SKILL.md +45 -0
- package/skills/check-spec-health/agents/openai.yaml +4 -0
- package/skills/competitive-positioning/SKILL.md +86 -0
- package/skills/competitive-positioning/agents/openai.yaml +4 -0
- package/skills/cpo-lens/SKILL.md +398 -0
- package/skills/cpo-lens/agents/openai.yaml +4 -0
- package/skills/create-prd/SKILL.md +26 -0
- package/skills/create-prd/agents/openai.yaml +4 -0
- package/skills/customer-discovery/SKILL.md +192 -0
- package/skills/design-system/SKILL.md +49 -0
- package/skills/design-system/agents/openai.yaml +4 -0
- package/skills/discussion-guide/SKILL.md +75 -0
- package/skills/discussion-guide/agents/openai.yaml +4 -0
- package/skills/firecrawl-research-papers/SKILL.md +96 -0
- package/skills/firecrawl-research-papers/agents/openai.yaml +4 -0
- package/skills/foo-roast/SKILL.md +325 -0
- package/skills/generate-gtm-brief/SKILL.md +41 -0
- package/skills/generate-gtm-brief/agents/openai.yaml +4 -0
- package/skills/generate-gtm-brief/references/gtm-brief-template.md +56 -0
- package/skills/gong-quotes/SKILL.md +43 -0
- package/skills/gong-quotes/agents/openai.yaml +4 -0
- package/skills/grill-me/SKILL.md +27 -0
- package/skills/grill-me/agents/openai.yaml +4 -0
- package/skills/ideate/SKILL.md +180 -0
- package/skills/ideate/references/family-packs.md +5 -0
- package/skills/initiative-tracker/SKILL.md +23 -0
- package/skills/initiative-tracker/agents/openai.yaml +4 -0
- package/skills/interop-review/SKILL.md +25 -0
- package/skills/interop-review/agents/openai.yaml +4 -0
- package/skills/interview-me/SKILL.md +404 -0
- package/skills/interview-me/agents/openai.yaml +4 -0
- package/skills/loop-library-audit/SKILL.md +123 -0
- package/skills/pitch/SKILL.md +110 -0
- package/skills/pitch/references/opportunity-assessment.md +136 -0
- package/skills/pitch/references/prfaq-template.md +54 -0
- package/skills/pmm-release-briefing/SKILL.md +27 -0
- package/skills/pmm-release-briefing/agents/openai.yaml +4 -0
- package/skills/pmm-release-briefing/references/variant-generalized.md +16 -0
- package/skills/positioning-messaging/SKILL.md +70 -0
- package/skills/positioning-messaging/agents/openai.yaml +4 -0
- package/skills/positioning-messaging/references/guest-insights.md +1400 -0
- package/skills/preview-swarm/SKILL.md +418 -0
- package/skills/preview-swarm/agents/openai.yaml +4 -0
- package/skills/preview-swarm/references/html-effectiveness-build-previews.md +168 -0
- package/skills/product-copywriting/SKILL.md +127 -0
- package/skills/product-copywriting/agents/openai.yaml +4 -0
- package/skills/product-copywriting/references/foo-voice.md +26 -0
- package/skills/product-shape-interview/SKILL.md +219 -0
- package/skills/product-shape-interview/agents/openai.yaml +4 -0
- package/skills/ready-for-release/SKILL.md +39 -0
- package/skills/ready-for-release/agents/openai.yaml +4 -0
- package/skills/redact-transcripts/SKILL.md +23 -0
- package/skills/redact-transcripts/agents/openai.yaml +4 -0
- package/skills/research-brief/SKILL.md +48 -0
- package/skills/research-brief/agents/openai.yaml +4 -0
- package/skills/research-refresh/SKILL.md +63 -0
- package/skills/research-refresh/references/worked-cases.md +44 -0
- package/skills/review-loop/SKILL.md +200 -0
- package/skills/review-spec/SKILL.md +32 -0
- package/skills/review-spec/agents/openai.yaml +4 -0
- package/skills/rfc-from-research/SKILL.md +42 -0
- package/skills/rfc-from-research/agents/openai.yaml +4 -0
- package/skills/rfc-from-research/references/rfc-pattern.md +69 -0
- package/skills/skill-registry.yaml +810 -0
- package/skills/spec-cleanup/SKILL.md +103 -0
- package/skills/spec-cleanup/agents/openai.yaml +4 -0
- package/skills/spec-swarm/SKILL.md +87 -0
- package/skills/spec-swarm/agents/openai.yaml +4 -0
- package/skills/spec-swarm/template/SKILL.template.md +94 -0
- package/skills/spec-swarm/template/personas/accessibility-inclusion.md +40 -0
- package/skills/spec-swarm/template/personas/ai-technical-pm.md +69 -0
- package/skills/spec-swarm/template/personas/analytics-engineering.md +39 -0
- package/skills/spec-swarm/template/personas/applied-ai-eval.md +64 -0
- package/skills/spec-swarm/template/personas/business-gm-pm.md +67 -0
- package/skills/spec-swarm/template/personas/consumer-craft-pm.md +68 -0
- package/skills/spec-swarm/template/personas/content-design.md +40 -0
- package/skills/spec-swarm/template/personas/cpo.md +42 -0
- package/skills/spec-swarm/template/personas/data-science.md +42 -0
- package/skills/spec-swarm/template/personas/developer-experience.md +65 -0
- package/skills/spec-swarm/template/personas/dx-product.md +70 -0
- package/skills/spec-swarm/template/personas/field.md +43 -0
- package/skills/spec-swarm/template/personas/finance-cfo.md +40 -0
- package/skills/spec-swarm/template/personas/growth-pm.md +67 -0
- package/skills/spec-swarm/template/personas/platform-pm.md +67 -0
- package/skills/spec-swarm/template/personas/pmm.md +43 -0
- package/skills/spec-swarm/template/personas/principal-product-designer.md +42 -0
- package/skills/spec-swarm/template/personas/product-ops-program.md +41 -0
- package/skills/spec-swarm/template/personas/qa-release.md +41 -0
- package/skills/spec-swarm/template/personas/responsible-ai-safety.md +64 -0
- package/skills/spec-swarm/template/personas/security-privacy-legal.md +40 -0
- package/skills/spec-swarm/template/personas/strategy-competitive.md +41 -0
- package/skills/spec-swarm/template/personas/technical-architect.md +42 -0
- package/skills/spec-swarm/template/personas/technical-lead.md +43 -0
- package/skills/spec-swarm/template/personas/user-research.md +41 -0
- package/skills/spec-swarm-accessibility-inclusion/SKILL.md +95 -0
- package/skills/spec-swarm-accessibility-inclusion/agents/openai.yaml +4 -0
- package/skills/spec-swarm-ai-technical-pm/SKILL.md +124 -0
- package/skills/spec-swarm-analytics-engineering/SKILL.md +94 -0
- package/skills/spec-swarm-analytics-engineering/agents/openai.yaml +4 -0
- package/skills/spec-swarm-applied-ai-eval/SKILL.md +120 -0
- package/skills/spec-swarm-business-gm-pm/SKILL.md +122 -0
- package/skills/spec-swarm-consumer-craft-pm/SKILL.md +122 -0
- package/skills/spec-swarm-content-design/SKILL.md +95 -0
- package/skills/spec-swarm-content-design/agents/openai.yaml +4 -0
- package/skills/spec-swarm-cpo/SKILL.md +94 -0
- package/skills/spec-swarm-cpo/agents/openai.yaml +4 -0
- package/skills/spec-swarm-data-science/SKILL.md +95 -0
- package/skills/spec-swarm-data-science/agents/openai.yaml +4 -0
- package/skills/spec-swarm-developer-experience/SKILL.md +121 -0
- package/skills/spec-swarm-dx-product/SKILL.md +126 -0
- package/skills/spec-swarm-field/SKILL.md +96 -0
- package/skills/spec-swarm-field/agents/openai.yaml +4 -0
- package/skills/spec-swarm-finance-cfo/SKILL.md +95 -0
- package/skills/spec-swarm-finance-cfo/agents/openai.yaml +4 -0
- package/skills/spec-swarm-growth-pm/SKILL.md +122 -0
- package/skills/spec-swarm-platform-pm/SKILL.md +122 -0
- package/skills/spec-swarm-pmm/SKILL.md +96 -0
- package/skills/spec-swarm-pmm/agents/openai.yaml +4 -0
- package/skills/spec-swarm-principal-product-designer/SKILL.md +95 -0
- package/skills/spec-swarm-principal-product-designer/agents/openai.yaml +4 -0
- package/skills/spec-swarm-product-ops-program/SKILL.md +96 -0
- package/skills/spec-swarm-product-ops-program/agents/openai.yaml +4 -0
- package/skills/spec-swarm-qa-release/SKILL.md +96 -0
- package/skills/spec-swarm-qa-release/agents/openai.yaml +4 -0
- package/skills/spec-swarm-responsible-ai-safety/SKILL.md +120 -0
- package/skills/spec-swarm-security-privacy-legal/SKILL.md +95 -0
- package/skills/spec-swarm-security-privacy-legal/agents/openai.yaml +4 -0
- package/skills/spec-swarm-strategy-competitive/SKILL.md +96 -0
- package/skills/spec-swarm-strategy-competitive/agents/openai.yaml +4 -0
- package/skills/spec-swarm-technical-architect/SKILL.md +95 -0
- package/skills/spec-swarm-technical-architect/agents/openai.yaml +4 -0
- package/skills/spec-swarm-technical-lead/SKILL.md +96 -0
- package/skills/spec-swarm-technical-lead/agents/openai.yaml +4 -0
- package/skills/spec-swarm-user-research/SKILL.md +95 -0
- package/skills/spec-swarm-user-research/agents/openai.yaml +4 -0
- package/skills/strategy-check/SKILL.md +170 -0
- package/skills/strategy-check/agents/openai.yaml +4 -0
- package/skills/study-digest/SKILL.md +41 -0
- package/skills/synthetic-customer/SKILL.md +68 -0
- package/skills/update-cli-spec/SKILL.md +69 -0
- package/skills/update-cli-spec/agents/openai.yaml +4 -0
- package/skills/voc-researcher/SKILL.md +45 -0
- package/skills/voc-researcher/agents/openai.yaml +4 -0
- package/skills-lock.json +35 -0
|
@@ -0,0 +1,405 @@
|
|
|
1
|
+
// `foo eval` — foo-eval-system-v0's front door.
|
|
2
|
+
//
|
|
3
|
+
// The contract, the deterministic checks, the judge runner and the calibration
|
|
4
|
+
// gate all shipped before this file existed, and none of them could be reached:
|
|
5
|
+
// no command imported them, so 891 lines of eval system had zero callers and
|
|
6
|
+
// never even made it into the packaged build. A capability nobody can invoke is
|
|
7
|
+
// indistinguishable from one that was never written.
|
|
8
|
+
//
|
|
9
|
+
// This is the surface that makes them real. It stays thin on purpose — every
|
|
10
|
+
// judgement lives in the modules it calls; this file parses arguments, resolves
|
|
11
|
+
// a packet, and renders.
|
|
12
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
13
|
+
import path from "node:path";
|
|
14
|
+
import { createInterface } from "node:readline";
|
|
15
|
+
import { parse as parseYaml } from "../vendorYaml.js";
|
|
16
|
+
import { UsageError } from "../errorTaxonomy.js";
|
|
17
|
+
import { EVAL_CORPUS_HOME, loadEvalSuite } from "../evalRecord.js";
|
|
18
|
+
import { compileSuite, runSuite } from "../evalCompile.js";
|
|
19
|
+
import { computeCalibration, judgeKey } from "../evalCalibration.js";
|
|
20
|
+
import { compileRoastRun, parseRoastRun, renderApprovalRequest, setApproval, writeCompiledSuite, } from "../evalRoast.js";
|
|
21
|
+
import { appendEvent } from "../packetWrite.js";
|
|
22
|
+
import { appendLabel, judgeSamplesPath, labelsPath, loadCorpusArtifacts, loadJudgeSamples, loadOwnerLabels, runCaptureSession, stratifiedSample, suiteArtifactsDir, } from "../evalLabels.js";
|
|
23
|
+
import { withGlyph } from "../statusGlyphs.js";
|
|
24
|
+
import { flagValue, resolvePacketDir } from "./shared.js";
|
|
25
|
+
import { EVAL_SUBCOMMANDS, EVAL_USAGE_LINES } from "../commandUsage.js";
|
|
26
|
+
function usage() {
|
|
27
|
+
return `${EVAL_USAGE_LINES.join("\n")}\n`;
|
|
28
|
+
}
|
|
29
|
+
/** Calibration reports live beside the records; absent means advisory-only. */
|
|
30
|
+
function calibrationResolver(packetDir) {
|
|
31
|
+
return (record) => {
|
|
32
|
+
const ref = record.judge?.calibration;
|
|
33
|
+
if (!ref)
|
|
34
|
+
return undefined;
|
|
35
|
+
const file = path.join(packetDir, "evals", ref);
|
|
36
|
+
if (!existsSync(file))
|
|
37
|
+
return undefined;
|
|
38
|
+
try {
|
|
39
|
+
return parseYaml(readFileSync(file, "utf8"));
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
return undefined;
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
export async function runEvalCommand(root, args, glyphs) {
|
|
47
|
+
const sub = args[0];
|
|
48
|
+
const json = args.includes("--json");
|
|
49
|
+
if (!sub || sub === "help" || sub === "--help")
|
|
50
|
+
return { output: usage(), exitCode: 0 };
|
|
51
|
+
if (!EVAL_SUBCOMMANDS.includes(sub)) {
|
|
52
|
+
throw new UsageError(`unknown eval subcommand: ${sub}`, "E_BAD_FLAG");
|
|
53
|
+
}
|
|
54
|
+
const { packetDir } = resolvePacketDir(root, args[1]);
|
|
55
|
+
if (sub === "validate") {
|
|
56
|
+
const result = loadEvalSuite(packetDir);
|
|
57
|
+
if (json)
|
|
58
|
+
return { output: `${JSON.stringify(result, null, 2)}\n`, exitCode: result.ok ? 0 : 1 };
|
|
59
|
+
if (result.records.length === 0) {
|
|
60
|
+
return { output: "no eval records in this packet — compile some with `foo eval compile`.\n", exitCode: 0 };
|
|
61
|
+
}
|
|
62
|
+
const approved = result.records.filter((record) => record.approved).length;
|
|
63
|
+
const lines = [
|
|
64
|
+
result.ok
|
|
65
|
+
? withGlyph("proven", `eval records ok: ${result.records.length} record(s), ${approved} approved.`, glyphs)
|
|
66
|
+
: withGlyph("blocked", `eval records refused: ${result.findings.length} finding(s).`, glyphs),
|
|
67
|
+
];
|
|
68
|
+
for (const finding of result.findings) {
|
|
69
|
+
lines.push(` ${finding.record} ${finding.field}: ${finding.message}`);
|
|
70
|
+
}
|
|
71
|
+
return { output: `${lines.join("\n")}\n`, exitCode: result.ok ? 0 : 1 };
|
|
72
|
+
}
|
|
73
|
+
if (sub === "compile") {
|
|
74
|
+
const roastPath = flagValue(args, "--roast");
|
|
75
|
+
if (!roastPath)
|
|
76
|
+
throw new UsageError("foo eval compile needs --roast <run.json>", "E_BAD_FLAG");
|
|
77
|
+
const file = path.resolve(root, roastPath);
|
|
78
|
+
if (!existsSync(file))
|
|
79
|
+
throw new UsageError(`no roast run record at ${roastPath}`, "E_BAD_FLAG");
|
|
80
|
+
// Malformed JSON in the --roast file is the caller's typo, not productfoo's bug:
|
|
81
|
+
// convert the raw SyntaxError to a UsageError so it exits as a usage error
|
|
82
|
+
// and files no Bug Capsule (#778). The read itself stays outside the catch so
|
|
83
|
+
// a filesystem failure still classifies as io_error.
|
|
84
|
+
let roastJson;
|
|
85
|
+
try {
|
|
86
|
+
roastJson = JSON.parse(readFileSync(file, "utf8"));
|
|
87
|
+
}
|
|
88
|
+
catch (error) {
|
|
89
|
+
throw new UsageError(`--roast file ${roastPath} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`, "E_BAD_FLAG");
|
|
90
|
+
}
|
|
91
|
+
const parsed = parseRoastRun(roastJson);
|
|
92
|
+
if (!parsed.run) {
|
|
93
|
+
const lines = parsed.findings.map((finding) => ` ${finding.message}`);
|
|
94
|
+
return {
|
|
95
|
+
output: `${withGlyph("blocked", "that is not a usable roast run record.", glyphs)}\n${lines.join("\n")}\n`,
|
|
96
|
+
exitCode: 1,
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
// Continue the packet's existing numbering so ids are minted once and never
|
|
100
|
+
// renumbered, even across several roasts of the same target.
|
|
101
|
+
const existing = loadEvalSuite(packetDir).records;
|
|
102
|
+
const highest = existing.reduce((max, record) => Math.max(max, Number(record.id.slice(2)) || 0), 0);
|
|
103
|
+
const compiled = compileRoastRun(parsed.run, path.basename(packetDir), { startAt: highest + 1 });
|
|
104
|
+
if (compiled.records.length > 0) {
|
|
105
|
+
writeCompiledSuite(packetDir, compiled.records);
|
|
106
|
+
appendEvent({
|
|
107
|
+
packetDir,
|
|
108
|
+
type: "eval_suite_compiled",
|
|
109
|
+
data: {
|
|
110
|
+
run: compiled.run,
|
|
111
|
+
compiled: compiled.records.map((record) => record.id),
|
|
112
|
+
refused: compiled.findings.length,
|
|
113
|
+
},
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
if (json)
|
|
117
|
+
return { output: `${JSON.stringify(compiled, null, 2)}\n`, exitCode: compiled.ok ? 0 : 1 };
|
|
118
|
+
return { output: `${renderApprovalRequest(compiled, glyphs)}\n`, exitCode: 0 };
|
|
119
|
+
}
|
|
120
|
+
if (sub === "approve" || sub === "reject") {
|
|
121
|
+
const id = flagValue(args, "--check");
|
|
122
|
+
const by = flagValue(args, "--by");
|
|
123
|
+
if (!id)
|
|
124
|
+
throw new UsageError(`foo eval ${sub} needs --check <E-xxx>`, "E_BAD_FLAG");
|
|
125
|
+
if (!by)
|
|
126
|
+
throw new UsageError(`foo eval ${sub} needs --by <who>`, "E_BAD_FLAG");
|
|
127
|
+
const reason = flagValue(args, "--reason");
|
|
128
|
+
if (sub === "reject" && !reason) {
|
|
129
|
+
// A rejection is early signal about the suite, so it has to say why.
|
|
130
|
+
throw new UsageError("foo eval reject needs --reason <why> — a rejection without a reason teaches nothing", "E_BAD_FLAG");
|
|
131
|
+
}
|
|
132
|
+
const updated = setApproval(packetDir, id, sub === "approve");
|
|
133
|
+
if (!updated) {
|
|
134
|
+
return { output: `${withGlyph("blocked", `no eval record ${id} in ${path.basename(packetDir)}.`, glyphs)}\n`, exitCode: 1 };
|
|
135
|
+
}
|
|
136
|
+
appendEvent({
|
|
137
|
+
packetDir,
|
|
138
|
+
type: sub === "approve" ? "eval_check_approved" : "eval_check_rejected",
|
|
139
|
+
actor: { type: "human", id: by },
|
|
140
|
+
data: { check: id, ...(reason ? { reason } : {}) },
|
|
141
|
+
});
|
|
142
|
+
const word = sub === "approve" ? "approved" : "rejected";
|
|
143
|
+
return {
|
|
144
|
+
output: `${withGlyph(sub === "approve" ? "proven" : "claimed", `${id} ${word} by ${by}.`, glyphs)}\n`,
|
|
145
|
+
exitCode: 0,
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
if (sub === "label") {
|
|
149
|
+
const suite = labelFlag(args, "--suite") ?? path.basename(packetDir);
|
|
150
|
+
const rawCount = labelFlag(args, "--count");
|
|
151
|
+
const count = rawCount === undefined ? 10 : Number(rawCount);
|
|
152
|
+
if (!Number.isInteger(count) || count <= 0) {
|
|
153
|
+
throw new UsageError("--count wants a positive whole number of artifacts (default 10)", "E_BAD_FLAG");
|
|
154
|
+
}
|
|
155
|
+
const judge = labelFlag(args, "--judge");
|
|
156
|
+
const dispute = labelFlag(args, "--dispute");
|
|
157
|
+
if (dispute !== undefined)
|
|
158
|
+
return relabelDisputed(root, packetDir, suite, dispute, judge, glyphs);
|
|
159
|
+
return captureBlindLabels(root, packetDir, suite, count, judge, glyphs);
|
|
160
|
+
}
|
|
161
|
+
// run
|
|
162
|
+
const targetArg = flagValue(args, "--target");
|
|
163
|
+
if (!targetArg)
|
|
164
|
+
throw new UsageError("foo eval run needs --target <file>", "E_BAD_FLAG");
|
|
165
|
+
const targetPath = path.resolve(root, targetArg);
|
|
166
|
+
if (!existsSync(targetPath))
|
|
167
|
+
throw new UsageError(`no artifact at ${targetArg}`, "E_BAD_FLAG");
|
|
168
|
+
const suite = loadEvalSuite(packetDir);
|
|
169
|
+
if (!suite.ok) {
|
|
170
|
+
return {
|
|
171
|
+
output: `${withGlyph("blocked", "refusing to run: the records do not validate. Run `foo eval validate`.", glyphs)}\n`,
|
|
172
|
+
exitCode: 1,
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
const compiled = compileSuite(suite.records);
|
|
176
|
+
const artifact = readFileSync(targetPath, "utf8");
|
|
177
|
+
const result = await runSuite(compiled, artifact, {
|
|
178
|
+
strict: args.includes("--strict"),
|
|
179
|
+
calibration: calibrationResolver(packetDir),
|
|
180
|
+
});
|
|
181
|
+
const ranCount = compiled.deterministic.length + compiled.judges.length;
|
|
182
|
+
if (json)
|
|
183
|
+
return { output: `${JSON.stringify({ ...result, ran: ranCount }, null, 2)}\n`, exitCode: ranCount === 0 ? 1 : result.ok ? 0 : 1 };
|
|
184
|
+
// A suite with nothing approved ran nothing, and "passes" would be a lie of
|
|
185
|
+
// exactly the kind this system exists to catch: a check that reports success
|
|
186
|
+
// when it reported nothing. Say what actually happened.
|
|
187
|
+
const ran = ranCount;
|
|
188
|
+
const lines = [
|
|
189
|
+
ran === 0
|
|
190
|
+
? withGlyph("needs_input", `no approved checks — nothing ran against ${targetArg}.`, glyphs)
|
|
191
|
+
: result.status === "advisory_only"
|
|
192
|
+
? withGlyph("claimed", `only advisory judges ran against ${targetArg}: ${ran} check(s); no gating checks evaluated.`, glyphs)
|
|
193
|
+
: result.ok
|
|
194
|
+
? withGlyph("proven", `eval suite passes on ${targetArg}: ${ran} check(s).`, glyphs)
|
|
195
|
+
: withGlyph("blocked", `eval suite fails on ${targetArg}.`, glyphs),
|
|
196
|
+
"",
|
|
197
|
+
];
|
|
198
|
+
for (const finding of result.deterministic.findings) {
|
|
199
|
+
lines.push(` ${finding.severity === "fail" ? "FAIL" : "warn"} ${finding.check}:${finding.line} ${finding.message}`);
|
|
200
|
+
}
|
|
201
|
+
for (const judge of result.judges) {
|
|
202
|
+
const tag = judge.advisory ? "advisory" : judge.verdict === "pass" ? "ok " : "FAIL";
|
|
203
|
+
lines.push(` ${tag} ${judge.id} ${judge.reason ?? judge.verdict}`);
|
|
204
|
+
if (judge.advisory)
|
|
205
|
+
lines.push(` ${judge.gating_reason}`);
|
|
206
|
+
}
|
|
207
|
+
if (result.skippedUnapproved.length > 0) {
|
|
208
|
+
lines.push("");
|
|
209
|
+
lines.push(`Not run — never approved: ${result.skippedUnapproved.join(", ")}`);
|
|
210
|
+
lines.push("Approve with: foo eval approve <packet-dir> --check <E-xxx> --by <who>");
|
|
211
|
+
}
|
|
212
|
+
return { output: `${lines.join("\n")}\n`, exitCode: ran === 0 ? 1 : result.ok ? 0 : 1 };
|
|
213
|
+
}
|
|
214
|
+
// ── foo eval label — blind capture (W-2) ────────────────────────────────────
|
|
215
|
+
/** `--flag=value` and `--flag value` both resolve; the blueprint's examples use the first. */
|
|
216
|
+
function labelFlag(args, name) {
|
|
217
|
+
const inline = args.find((arg) => arg.startsWith(`${name}=`));
|
|
218
|
+
if (inline !== undefined)
|
|
219
|
+
return inline.slice(name.length + 1);
|
|
220
|
+
return flagValue(args, name);
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Line reader for the capture session. Works piped (each line an answer) and
|
|
224
|
+
* on a terminal (line-buffered); end of input is null, never an answer.
|
|
225
|
+
*/
|
|
226
|
+
function stdinLineReader() {
|
|
227
|
+
const rl = createInterface({ input: process.stdin, terminal: false });
|
|
228
|
+
const iterator = rl[Symbol.asyncIterator]();
|
|
229
|
+
let closed = false;
|
|
230
|
+
return async () => {
|
|
231
|
+
if (closed)
|
|
232
|
+
return null;
|
|
233
|
+
const next = await iterator.next();
|
|
234
|
+
if (next.done) {
|
|
235
|
+
closed = true;
|
|
236
|
+
rl.close();
|
|
237
|
+
return null;
|
|
238
|
+
}
|
|
239
|
+
return next.value;
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
async function captureBlindLabels(root, packetDir, suite, count, judgeFlag, glyphs) {
|
|
243
|
+
const artifacts = loadCorpusArtifacts(root, suite);
|
|
244
|
+
if (artifacts.length === 0) {
|
|
245
|
+
return {
|
|
246
|
+
output: `${withGlyph("blocked", `no corpus artifacts for suite ${suite} — nothing to label.`, glyphs)}\nArtifacts live under ${suiteArtifactsDir(".", suite)} and never enter git (R-008).\n`,
|
|
247
|
+
exitCode: 1,
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
const load = loadOwnerLabels(root, suite);
|
|
251
|
+
const labeled = new Set(load.current.map((label) => label.artifact));
|
|
252
|
+
const remaining = artifacts.filter((view) => !labeled.has(view.hash));
|
|
253
|
+
if (remaining.length === 0) {
|
|
254
|
+
const lines = [
|
|
255
|
+
withGlyph("needs_input", `every artifact in suite ${suite} already has a blind label — nothing to sample.`, glyphs),
|
|
256
|
+
`Relabel after a dispute: foo eval label ${path.basename(packetDir)} --dispute <hash-prefix>`,
|
|
257
|
+
...reportLines(root, packetDir, suite, judgeFlag, glyphs),
|
|
258
|
+
];
|
|
259
|
+
return { output: `${lines.join("\n")}\n`, exitCode: 0 };
|
|
260
|
+
}
|
|
261
|
+
const sampled = stratifiedSample(remaining, (view) => view.shape, count);
|
|
262
|
+
// Each label lands the moment it is answered — a Ctrl-C mid-session keeps
|
|
263
|
+
// everything captured so far instead of losing the batch.
|
|
264
|
+
const { labels, abandoned } = await runCaptureSession(suite, sampled, {
|
|
265
|
+
readLine: stdinLineReader(),
|
|
266
|
+
write: (text) => process.stdout.write(text),
|
|
267
|
+
}, {
|
|
268
|
+
origin: "capture",
|
|
269
|
+
onLabel: (label) => appendLabel(root, label),
|
|
270
|
+
});
|
|
271
|
+
const lines = [];
|
|
272
|
+
if (load.malformed > 0) {
|
|
273
|
+
lines.push(withGlyph("drift", `${load.malformed} malformed label line(s) ignored — the file is never repaired in place.`, glyphs));
|
|
274
|
+
}
|
|
275
|
+
if (labels.length === 0) {
|
|
276
|
+
lines.push(withGlyph("blocked", "nothing recorded — an empty capture is not a labeling session.", glyphs));
|
|
277
|
+
return { output: `${lines.join("\n")}\n`, exitCode: 1 };
|
|
278
|
+
}
|
|
279
|
+
const pass = labels.filter((label) => label.verdict === "pass").length;
|
|
280
|
+
lines.push(withGlyph("proven", `recorded ${labels.length} blind label(s) for ${suite}: ${pass} pass, ${labels.length - pass} fail.`, glyphs));
|
|
281
|
+
lines.push(` → ${path.relative(root, labelsPath(root))} — private corpus, never in git, never packed (R-008).`);
|
|
282
|
+
if (sampled.length < count) {
|
|
283
|
+
lines.push(` asked for ${count}; ${remaining.length} unlabeled artifact(s) available.`);
|
|
284
|
+
}
|
|
285
|
+
if (abandoned) {
|
|
286
|
+
lines.push(`Input ended after ${labels.length} of ${sampled.length} — rerun \`foo eval label ${path.basename(packetDir)}\` to continue; the remainder stays available.`);
|
|
287
|
+
}
|
|
288
|
+
lines.push(...reportLines(root, packetDir, suite, judgeFlag, glyphs));
|
|
289
|
+
return { output: `${lines.join("\n")}\n`, exitCode: 0 };
|
|
290
|
+
}
|
|
291
|
+
async function relabelDisputed(root, packetDir, suite, dispute, judgeFlag, glyphs) {
|
|
292
|
+
const prefix = dispute.trim().toLowerCase();
|
|
293
|
+
const matches = loadCorpusArtifacts(root, suite).filter((view) => view.hash.startsWith(prefix));
|
|
294
|
+
if (matches.length === 0) {
|
|
295
|
+
return {
|
|
296
|
+
output: `${withGlyph("blocked", `no corpus artifact matches hash prefix "${dispute}" in suite ${suite}.`, glyphs)}\nLabels key on content hashes; the artifact bytes must exist under ${EVAL_CORPUS_HOME}/${suite}/artifacts/ to present blind.\n`,
|
|
297
|
+
exitCode: 1,
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
if (matches.length > 1) {
|
|
301
|
+
return {
|
|
302
|
+
output: `${withGlyph("blocked", `hash prefix "${dispute}" matches ${matches.length} artifacts — use a longer prefix.`, glyphs)}\n${matches.map((match) => ` ${match.hash.slice(0, 12)} ${match.file}`).join("\n")}\n`,
|
|
303
|
+
exitCode: 1,
|
|
304
|
+
};
|
|
305
|
+
}
|
|
306
|
+
const view = matches[0];
|
|
307
|
+
const { labels } = await runCaptureSession(suite, [view], {
|
|
308
|
+
readLine: stdinLineReader(),
|
|
309
|
+
write: (text) => process.stdout.write(text),
|
|
310
|
+
}, {
|
|
311
|
+
origin: "dispute",
|
|
312
|
+
provenance: `dispute relabel via foo eval label --dispute (${view.hash.slice(0, 12)})`,
|
|
313
|
+
onLabel: (label) => appendLabel(root, label),
|
|
314
|
+
});
|
|
315
|
+
if (labels.length === 0) {
|
|
316
|
+
return { output: `${withGlyph("blocked", "nothing recorded — the relabel needs a verdict.", glyphs)}\n`, exitCode: 1 };
|
|
317
|
+
}
|
|
318
|
+
const label = labels[0];
|
|
319
|
+
const lines = [
|
|
320
|
+
withGlyph("proven", `relabeled ${label.artifact.slice(0, 12)} (${view.file}) as ${label.verdict}, blind — the relabel supersedes by later append; history is kept.`, glyphs),
|
|
321
|
+
` → ${path.relative(root, labelsPath(root))}`,
|
|
322
|
+
...reportLines(root, packetDir, suite, judgeFlag, glyphs),
|
|
323
|
+
];
|
|
324
|
+
return { output: `${lines.join("\n")}\n`, exitCode: 0 };
|
|
325
|
+
}
|
|
326
|
+
// ── the calibration section the label verb prints after a session ───────────
|
|
327
|
+
/** Lines to append to a successful label-verb run: the calibration report. */
|
|
328
|
+
function reportLines(root, packetDir, suite, judgeFlag, glyphs) {
|
|
329
|
+
const section = calibrationSection(root, packetDir, suite, judgeFlag, glyphs);
|
|
330
|
+
if (section.blocked)
|
|
331
|
+
return [` calibration report not shown: ${section.blocked}`];
|
|
332
|
+
return section.lines;
|
|
333
|
+
}
|
|
334
|
+
/**
|
|
335
|
+
* Resolve which judge record the report is about: --judge names one explicitly;
|
|
336
|
+
* a unique judge record for the suite stands in on its own. The report is
|
|
337
|
+
* informational — a mis-named judge can never undo a recorded label, so it
|
|
338
|
+
* downgrades to a note instead of failing the capture.
|
|
339
|
+
*/
|
|
340
|
+
function calibrationSection(root, packetDir, suite, judgeFlag, glyphs) {
|
|
341
|
+
const records = loadEvalSuite(packetDir).records.filter((record) => record.kind === "judge" && record.suite === suite);
|
|
342
|
+
if (judgeFlag !== undefined) {
|
|
343
|
+
const named = records.find((record) => record.id.toUpperCase() === judgeFlag.toUpperCase());
|
|
344
|
+
if (!named) {
|
|
345
|
+
return {
|
|
346
|
+
blocked: `no judge record ${judgeFlag} for suite ${suite} — --judge names a judge record in ${path.basename(packetDir)}.`,
|
|
347
|
+
lines: [],
|
|
348
|
+
};
|
|
349
|
+
}
|
|
350
|
+
return judgeReportSection(root, named, suite, glyphs);
|
|
351
|
+
}
|
|
352
|
+
if (records.length === 1)
|
|
353
|
+
return judgeReportSection(root, records[0], suite, glyphs);
|
|
354
|
+
if (records.length === 0)
|
|
355
|
+
return { lines: [] };
|
|
356
|
+
return {
|
|
357
|
+
blocked: `suite ${suite} has ${records.length} judge records (${records.map((record) => record.id).join(", ")}) — name one with --judge <E-xxx>.`,
|
|
358
|
+
lines: [],
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
/** The live report for one judge record: the blueprint's "then print the
|
|
362
|
+
* calibration report". Needs stored judge samples; without them it says
|
|
363
|
+
* exactly what is missing instead of pretending the math ran. */
|
|
364
|
+
function judgeReportSection(root, record, suite, glyphs) {
|
|
365
|
+
if (!record.judge) {
|
|
366
|
+
return { blocked: `record ${record.id} carries no judge spec.`, lines: [] };
|
|
367
|
+
}
|
|
368
|
+
const judge = record.judge;
|
|
369
|
+
const { samples: stored, malformed } = loadJudgeSamples(root, suite);
|
|
370
|
+
const key = judgeKey(judge);
|
|
371
|
+
const current = stored.filter((sample) => sample.judge_key === key);
|
|
372
|
+
const drifted = stored.length - current.length;
|
|
373
|
+
if (current.length === 0) {
|
|
374
|
+
const lines = [
|
|
375
|
+
"",
|
|
376
|
+
`no judge samples stored under this judge_key — the calibration report needs judge verdicts keyed by artifact hash.`,
|
|
377
|
+
` samples live at ${path.relative(root, judgeSamplesPath(root, suite))} (private corpus, R-008).`,
|
|
378
|
+
];
|
|
379
|
+
if (malformed > 0)
|
|
380
|
+
lines.push(` ${malformed} malformed sample line(s) ignored.`);
|
|
381
|
+
if (drifted > 0)
|
|
382
|
+
lines.push(` ${drifted} sample(s) from an earlier judge_key excluded — drift voids them until revalidation against the same labels.`);
|
|
383
|
+
return { lines };
|
|
384
|
+
}
|
|
385
|
+
const { current: labels, malformed: labelMalformed } = loadOwnerLabels(root, suite);
|
|
386
|
+
const report = computeCalibration(judge, labels, current);
|
|
387
|
+
const lines = [
|
|
388
|
+
"",
|
|
389
|
+
`calibration for ${record.id} (${judge.model}): ${report.certified ? "certified" : "not certified"} — ${report.labels.total} blind label(s), TPR ${report.tpr ?? "—"}, TNR ${report.tnr ?? "—"}.`,
|
|
390
|
+
` kappa ${report.kappa ?? "—"} (chance-corrected owner/judge agreement — diagnostic, never gating).`,
|
|
391
|
+
];
|
|
392
|
+
if (labelMalformed > 0)
|
|
393
|
+
lines.push(` ${labelMalformed} malformed label line(s) ignored.`);
|
|
394
|
+
for (const reason of report.reasons)
|
|
395
|
+
lines.push(` · ${reason}`);
|
|
396
|
+
if (report.disputes.length > 0) {
|
|
397
|
+
lines.push(" dispute relabels — refinement signals for the criterion or the judge prompt (never gate inputs):");
|
|
398
|
+
for (const dispute of report.disputes) {
|
|
399
|
+
const judged = dispute.judge_verdict ? `judge ${dispute.judge_verdict}` : "no judge sample";
|
|
400
|
+
lines.push(` ${dispute.artifact.slice(0, 12)} owner ${dispute.owner_verdict} · ${judged}${dispute.note ? ` — "${dispute.note}"` : ""}`);
|
|
401
|
+
}
|
|
402
|
+
lines.push(" a dispute asks for a closer look: refine the record's criterion, or the judge prompt — an edited prompt changes the judge_key and voids calibration until revalidated against the same labels.");
|
|
403
|
+
}
|
|
404
|
+
return { lines };
|
|
405
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
export type VerifyResult = {
|
|
2
|
+
state: "valid";
|
|
3
|
+
} | {
|
|
4
|
+
state: "rejected";
|
|
5
|
+
} | {
|
|
6
|
+
state: "unreachable";
|
|
7
|
+
detail: string;
|
|
8
|
+
};
|
|
9
|
+
export type LoginOutcome = {
|
|
10
|
+
output: string;
|
|
11
|
+
exitCode: number;
|
|
12
|
+
};
|
|
13
|
+
export declare function verifyFirecrawlKey(key: string, baseUrl: string, timeoutMs?: number): Promise<VerifyResult>;
|
|
14
|
+
export declare function firecrawlLoginCommand(key: string | undefined, opts?: {
|
|
15
|
+
force?: boolean;
|
|
16
|
+
env?: NodeJS.ProcessEnv;
|
|
17
|
+
machine?: boolean;
|
|
18
|
+
args?: string[];
|
|
19
|
+
}): Promise<LoginOutcome>;
|
|
20
|
+
export declare function firecrawlLogoutCommand(env?: NodeJS.ProcessEnv): LoginOutcome;
|
|
21
|
+
export declare function firecrawlKeyStatus(env?: NodeJS.ProcessEnv): {
|
|
22
|
+
present: boolean;
|
|
23
|
+
source: string;
|
|
24
|
+
detail: string;
|
|
25
|
+
};
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
// Direct Firecrawl credentials are verified with Firecrawl and stored locally.
|
|
2
|
+
import { clearStoredFirecrawlKey, credentialsPath, resolveFirecrawlKey, writeStoredFirecrawlKey, } from "../tokenStore.js";
|
|
3
|
+
function tokenFingerprint(token) {
|
|
4
|
+
return token.slice(-4);
|
|
5
|
+
}
|
|
6
|
+
export async function verifyFirecrawlKey(key, baseUrl, timeoutMs = 10_000) {
|
|
7
|
+
let res;
|
|
8
|
+
try {
|
|
9
|
+
res = await fetch(`${baseUrl.replace(/\/+$/, "")}/v1/team/credit-usage`, {
|
|
10
|
+
headers: { authorization: `Bearer ${key}` },
|
|
11
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
12
|
+
redirect: "error",
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
catch (err) {
|
|
16
|
+
return { state: "unreachable", detail: String(err) };
|
|
17
|
+
}
|
|
18
|
+
if (res.status === 401 || res.status === 403)
|
|
19
|
+
return { state: "rejected" };
|
|
20
|
+
if (res.ok || res.status === 404)
|
|
21
|
+
return { state: "valid" };
|
|
22
|
+
return { state: "unreachable", detail: `HTTP ${res.status}` };
|
|
23
|
+
}
|
|
24
|
+
export async function firecrawlLoginCommand(key, opts = {}) {
|
|
25
|
+
const env = opts.env ?? process.env;
|
|
26
|
+
const trimmed = (key ?? "").trim();
|
|
27
|
+
if (!trimmed) {
|
|
28
|
+
return {
|
|
29
|
+
output: "usage: foo login --firecrawl <key> (your own Firecrawl API key)\n" +
|
|
30
|
+
" foo logout --firecrawl (forget it)\n",
|
|
31
|
+
exitCode: 1,
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
const base = (env.FIRECRAWL_BASE_URL || "").trim() || "https://api.firecrawl.dev";
|
|
35
|
+
const fp = tokenFingerprint(trimmed);
|
|
36
|
+
if (!opts.force) {
|
|
37
|
+
const result = await verifyFirecrawlKey(trimmed, base);
|
|
38
|
+
if (result.state === "rejected") {
|
|
39
|
+
return {
|
|
40
|
+
output: `✋ needs your input — Firecrawl rejected that key (…${fp}). Nothing was saved.\n` +
|
|
41
|
+
"Check you pasted the whole key from firecrawl.dev; it usually starts with fc-.\n",
|
|
42
|
+
exitCode: 1,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
if (result.state === "unreachable") {
|
|
46
|
+
return {
|
|
47
|
+
output: "🚧 blocked — couldn't reach Firecrawl to check that key, so it was NOT saved.\n" +
|
|
48
|
+
"Your key was not rejected; this is a network problem. Retry, or save it unchecked with:\n" +
|
|
49
|
+
" foo login --firecrawl <key> --force\n",
|
|
50
|
+
exitCode: 1,
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
const file = writeStoredFirecrawlKey(trimmed, env);
|
|
55
|
+
const lines = [
|
|
56
|
+
opts.force
|
|
57
|
+
? `✅ saved — Firecrawl key (…${fp}) stored unchecked at ${file} (0600).`
|
|
58
|
+
: `✅ proven — Firecrawl key (…${fp}) accepted by Firecrawl and saved to ${file} (0600).`,
|
|
59
|
+
"Direct Firecrawl access is active; public-web output remains untrusted and is not research proof.",
|
|
60
|
+
];
|
|
61
|
+
const shadowed = (env.FIRECRAWL_API_KEY ?? "").trim();
|
|
62
|
+
if (shadowed && shadowed !== trimmed) {
|
|
63
|
+
lines.push("", `⚠️ drift — FIRECRAWL_API_KEY is also set in this shell (…${tokenFingerprint(shadowed)}) and it WINS over the saved one.`, "Unset it here — `unset FIRECRAWL_API_KEY` — for the key you just saved to take effect.");
|
|
64
|
+
}
|
|
65
|
+
return { output: `${lines.join("\n")}\n`, exitCode: 0 };
|
|
66
|
+
}
|
|
67
|
+
export function firecrawlLogoutCommand(env = process.env) {
|
|
68
|
+
const file = credentialsPath(env);
|
|
69
|
+
const removed = clearStoredFirecrawlKey(env);
|
|
70
|
+
const stillEnv = (env.FIRECRAWL_API_KEY ?? "").trim();
|
|
71
|
+
const lines = removed ? [`✅ proven — forgot the saved Firecrawl key (${file}).`] : ["📋 todo — no saved Firecrawl key to forget."];
|
|
72
|
+
if (stillEnv) {
|
|
73
|
+
lines.push(`⚠️ drift — FIRECRAWL_API_KEY remains set in this shell (…${tokenFingerprint(stillEnv)}), so direct Firecrawl access remains active.`);
|
|
74
|
+
}
|
|
75
|
+
return { output: `${lines.join("\n")}\n`, exitCode: 0 };
|
|
76
|
+
}
|
|
77
|
+
// Provider availability is inventory, not research proof.
|
|
78
|
+
export function firecrawlKeyStatus(env = process.env) {
|
|
79
|
+
const { key, source } = resolveFirecrawlKey(env);
|
|
80
|
+
if (!key) {
|
|
81
|
+
return {
|
|
82
|
+
present: false,
|
|
83
|
+
source: "none",
|
|
84
|
+
detail: "no Firecrawl credential — web research uses the free path",
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
return {
|
|
88
|
+
present: true,
|
|
89
|
+
source,
|
|
90
|
+
detail: source === "env"
|
|
91
|
+
? `direct Firecrawl access from FIRECRAWL_API_KEY in this shell (…${tokenFingerprint(key)}) — this shell only; output remains untrusted`
|
|
92
|
+
: `direct Firecrawl access from ${credentialsPath(env)} (…${tokenFingerprint(key)}) — every shell on this machine; output remains untrusted`,
|
|
93
|
+
};
|
|
94
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { estimateSpecBudget } from "../specBudget.js";
|
|
2
|
+
export declare function runLoopCommand(root: string, command: string | undefined, packetArg: string | undefined, args: string[]): Promise<string>;
|
|
3
|
+
export declare function runLoopRoutine(root: string, packetDir: string, args: string[]): {
|
|
4
|
+
output: string;
|
|
5
|
+
exitCode: number;
|
|
6
|
+
};
|
|
7
|
+
export declare function runSpecEstimate(root: string, packetDir: string, packetLabel: string, args: string[]): Promise<string>;
|
|
8
|
+
export declare function costHeadline(root: string, packetDir: string, opts?: {
|
|
9
|
+
remainingOnly?: boolean;
|
|
10
|
+
}): Promise<string | null>;
|
|
11
|
+
export declare function renderPerTask(estimate: Awaited<ReturnType<typeof estimateSpecBudget>>, tasks: {
|
|
12
|
+
id: string;
|
|
13
|
+
title: string;
|
|
14
|
+
status: string;
|
|
15
|
+
refs: string[];
|
|
16
|
+
}[]): string;
|