stageflow 0.40.0 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -4
- package/dist/agent/activity.d.ts +18 -0
- package/dist/agent/claudeAdapter.js +7 -0
- package/dist/agent/claudeTools.d.ts +27 -18
- package/dist/agent/claudeTools.js +54 -15
- package/dist/agent/compactionEvents.d.ts +34 -0
- package/dist/agent/compactionEvents.js +37 -0
- package/dist/agent/openRouterRouting.d.ts +7 -0
- package/dist/agent/openRouterRouting.js +23 -0
- package/dist/agent/piAdapter.d.ts +6 -0
- package/dist/agent/piAdapter.js +128 -27
- package/dist/agent/piSettings.d.ts +7 -0
- package/dist/agent/piSettings.js +9 -0
- package/dist/agent/sandboxPiTools.d.ts +3 -1
- package/dist/agent/sandboxPiTools.js +8 -2
- package/dist/agent/stageTurnDriver.d.ts +11 -2
- package/dist/agent/stageTurnDriver.js +45 -4
- package/dist/agent/toolOutputBudget.d.ts +15 -1
- package/dist/agent/toolOutputBudget.js +18 -0
- package/dist/agent/toolOutputBudgetExtension.d.ts +2 -0
- package/dist/agent/toolOutputBudgetExtension.js +40 -3
- package/dist/agents/authorChat.d.ts +5 -0
- package/dist/agents/authorChat.js +120 -12
- package/dist/agents/authorSessionStore.d.ts +38 -0
- package/dist/agents/authorSessionStore.js +167 -0
- package/dist/agents/chatTurn.d.ts +2 -0
- package/dist/agents/chatTurn.js +17 -1
- package/dist/agents/errors.d.ts +1 -1
- package/dist/agents/errors.js +1 -0
- package/dist/agents/evalGate.d.ts +61 -0
- package/dist/agents/evalGate.js +103 -0
- package/dist/agents/evalSession.d.ts +26 -0
- package/dist/agents/evalSession.js +29 -0
- package/dist/agents/hostContext.d.ts +4 -1
- package/dist/agents/hostContext.js +4 -1
- package/dist/agents/runtime/piRuntime.d.ts +6 -2
- package/dist/agents/runtime/piRuntime.js +50 -16
- package/dist/agents/runtime/types.d.ts +3 -1
- package/dist/agents/service.d.ts +27 -2
- package/dist/agents/service.js +91 -15
- package/dist/agents/sessions.d.ts +8 -0
- package/dist/agents/sessions.js +53 -11
- package/dist/agents/store.d.ts +14 -3
- package/dist/agents/store.js +45 -7
- package/dist/agents/toolRisk.d.ts +21 -0
- package/dist/agents/toolRisk.js +50 -0
- package/dist/cli/doctorCommand.js +26 -0
- package/dist/cli/evalCommand.d.ts +20 -0
- package/dist/cli/evalCommand.js +720 -0
- package/dist/cli/exportAllCommand.js +1 -1
- package/dist/cli/hostClient.d.ts +60 -0
- package/dist/cli/hostClient.js +71 -0
- package/dist/cli/validateOutput.d.ts +2 -2
- package/dist/cli/validateOutput.js +2 -2
- package/dist/cli.js +10 -0
- package/dist/compaction/directives.d.ts +18 -0
- package/dist/compaction/directives.js +119 -0
- package/dist/compaction/extension.d.ts +20 -0
- package/dist/compaction/extension.js +78 -0
- package/dist/compaction/mask.d.ts +18 -0
- package/dist/compaction/mask.js +155 -0
- package/dist/compaction/pin.d.ts +21 -0
- package/dist/compaction/pin.js +270 -0
- package/dist/compaction/scrub.d.ts +7 -0
- package/dist/compaction/scrub.js +26 -0
- package/dist/compaction/sessionWiring.d.ts +22 -0
- package/dist/compaction/sessionWiring.js +26 -0
- package/dist/compaction/summarize.d.ts +79 -0
- package/dist/compaction/summarize.js +171 -0
- package/dist/compaction/toolPolicy.d.ts +3 -0
- package/dist/compaction/toolPolicy.js +33 -0
- package/dist/compaction/types.d.ts +45 -0
- package/dist/compaction/types.js +7 -0
- package/dist/compaction/workingState.d.ts +56 -0
- package/dist/compaction/workingState.js +198 -0
- package/dist/config/atomicWrite.d.ts +15 -0
- package/dist/config/atomicWrite.js +43 -0
- package/dist/config/browseCatalog.d.ts +2 -0
- package/dist/config/browseCatalog.js +4 -2
- package/dist/config/createPipeline.js +2 -0
- package/dist/config/createTrigger.js +7 -1
- package/dist/config/csv.d.ts +9 -0
- package/dist/config/csv.js +81 -0
- package/dist/config/datasetAgentCheck.d.ts +18 -0
- package/dist/config/datasetAgentCheck.js +109 -0
- package/dist/config/datasetCandidates.d.ts +22 -0
- package/dist/config/datasetCandidates.js +86 -0
- package/dist/config/datasetFromSessions.d.ts +34 -0
- package/dist/config/datasetFromSessions.js +105 -0
- package/dist/config/datasetHash.d.ts +1 -0
- package/dist/config/datasetHash.js +4 -0
- package/dist/config/datasetImport.d.ts +21 -0
- package/dist/config/datasetImport.js +120 -0
- package/dist/config/datasetStageCheck.d.ts +9 -0
- package/dist/config/datasetStageCheck.js +17 -0
- package/dist/config/datasetTypes.d.ts +27 -0
- package/dist/config/datasetTypes.js +1 -0
- package/dist/config/ensureCatalogScanRoot.d.ts +4 -1
- package/dist/config/ensureCatalogScanRoot.js +8 -6
- package/dist/config/evalCatalog.d.ts +22 -0
- package/dist/config/evalCatalog.js +95 -0
- package/dist/config/evalWriteRoots.d.ts +17 -0
- package/dist/config/evalWriteRoots.js +109 -0
- package/dist/config/githubTriggerAction.d.ts +8 -0
- package/dist/config/githubTriggerAction.js +29 -0
- package/dist/config/hostConfig.js +3 -0
- package/dist/config/installEvalSample.d.ts +34 -0
- package/dist/config/installEvalSample.js +182 -0
- package/dist/config/loadDataset.d.ts +16 -0
- package/dist/config/loadDataset.js +318 -0
- package/dist/config/loadEval.d.ts +32 -0
- package/dist/config/loadEval.js +1103 -0
- package/dist/config/loadOutcome.d.ts +5 -0
- package/dist/config/loadStageflowManifest.js +64 -0
- package/dist/config/pipelineNeeds.d.ts +2 -0
- package/dist/config/pipelineNeeds.js +16 -3
- package/dist/config/pipelineRoute.js +5 -4
- package/dist/config/resolveAgentTarget.d.ts +32 -0
- package/dist/config/resolveAgentTarget.js +118 -0
- package/dist/config/resolveEvalTarget.d.ts +21 -0
- package/dist/config/resolveEvalTarget.js +89 -0
- package/dist/config/scanCatalogPaths.d.ts +1 -1
- package/dist/config/scanCatalogPaths.js +17 -5
- package/dist/config/validateCatalog.d.ts +2 -2
- package/dist/config/validateCatalog.js +21 -1
- package/dist/config/validateEval.d.ts +36 -0
- package/dist/config/validateEval.js +446 -0
- package/dist/config/writeDataset.d.ts +54 -0
- package/dist/config/writeDataset.js +186 -0
- package/dist/config/writeEval.d.ts +25 -0
- package/dist/config/writeEval.js +398 -0
- package/dist/diagnostics/checks.js +1 -1
- package/dist/envelope/assertEmittedArtifacts.d.ts +12 -0
- package/dist/envelope/assertEmittedArtifacts.js +43 -0
- package/dist/envelope/preEmitChecks.d.ts +3 -0
- package/dist/evals/agentDiff.d.ts +13 -0
- package/dist/evals/agentDiff.js +11 -0
- package/dist/evals/canonical.d.ts +3 -0
- package/dist/evals/canonical.js +24 -0
- package/dist/evals/caseTask.d.ts +3 -0
- package/dist/evals/caseTask.js +9 -0
- package/dist/evals/compare.d.ts +94 -0
- package/dist/evals/compare.js +201 -0
- package/dist/evals/constants.d.ts +62 -0
- package/dist/evals/constants.js +97 -0
- package/dist/evals/engine/FakeEngine.d.ts +33 -0
- package/dist/evals/engine/FakeEngine.js +177 -0
- package/dist/evals/engine/index.d.ts +9 -0
- package/dist/evals/engine/index.js +19 -0
- package/dist/evals/engine/promptfoo/PromptfooEngine.d.ts +52 -0
- package/dist/evals/engine/promptfoo/PromptfooEngine.js +312 -0
- package/dist/evals/engine/promptfoo/errors.d.ts +5 -0
- package/dist/evals/engine/promptfoo/errors.js +8 -0
- package/dist/evals/engine/promptfoo/generate.d.ts +30 -0
- package/dist/evals/engine/promptfoo/generate.js +345 -0
- package/dist/evals/engine/promptfoo/guard.d.ts +10 -0
- package/dist/evals/engine/promptfoo/guard.js +116 -0
- package/dist/evals/engine/promptfoo/install.d.ts +17 -0
- package/dist/evals/engine/promptfoo/install.js +73 -0
- package/dist/evals/engine/promptfoo/judgeProviders.d.ts +2 -0
- package/dist/evals/engine/promptfoo/judgeProviders.js +9 -0
- package/dist/evals/engine/promptfoo/judgeServices.d.ts +6 -0
- package/dist/evals/engine/promptfoo/judgeServices.js +41 -0
- package/dist/evals/engine/promptfoo/locate.d.ts +21 -0
- package/dist/evals/engine/promptfoo/locate.js +68 -0
- package/dist/evals/engine/promptfoo/nodeVersion.d.ts +7 -0
- package/dist/evals/engine/promptfoo/nodeVersion.js +14 -0
- package/dist/evals/engine/promptfoo/normalize.d.ts +66 -0
- package/dist/evals/engine/promptfoo/normalize.js +261 -0
- package/dist/evals/engine/promptfoo/pinned.d.ts +5 -0
- package/dist/evals/engine/promptfoo/pinned.js +5 -0
- package/dist/evals/engine/promptfoo/rubricPrompt.d.ts +14 -0
- package/dist/evals/engine/promptfoo/rubricPrompt.js +105 -0
- package/dist/evals/engine/promptfoo/runTokens.d.ts +18 -0
- package/dist/evals/engine/promptfoo/runTokens.js +31 -0
- package/dist/evals/engine/promptfoo/status.d.ts +13 -0
- package/dist/evals/engine/promptfoo/status.js +16 -0
- package/dist/evals/estimate.d.ts +56 -0
- package/dist/evals/estimate.js +66 -0
- package/dist/evals/graders/codeGrader.d.ts +6 -0
- package/dist/evals/graders/codeGrader.js +85 -0
- package/dist/evals/graders/conversationCheckGrader.d.ts +6 -0
- package/dist/evals/graders/conversationCheckGrader.js +62 -0
- package/dist/evals/graders/fieldGrader.d.ts +11 -0
- package/dist/evals/graders/fieldGrader.js +117 -0
- package/dist/evals/graders/goal.d.ts +3 -0
- package/dist/evals/graders/goal.js +14 -0
- package/dist/evals/graders/index.d.ts +18 -0
- package/dist/evals/graders/index.js +34 -0
- package/dist/evals/graders/notApplicable.d.ts +5 -0
- package/dist/evals/graders/notApplicable.js +9 -0
- package/dist/evals/graders/path.d.ts +26 -0
- package/dist/evals/graders/path.js +75 -0
- package/dist/evals/graders/referenceGrader.d.ts +5 -0
- package/dist/evals/graders/referenceGrader.js +59 -0
- package/dist/evals/graders/schemaGrader.d.ts +5 -0
- package/dist/evals/graders/schemaGrader.js +29 -0
- package/dist/evals/graders/subject.d.ts +6 -0
- package/dist/evals/graders/subject.js +33 -0
- package/dist/evals/graders/toolCallsGrader.d.ts +6 -0
- package/dist/evals/graders/toolCallsGrader.js +100 -0
- package/dist/evals/graders/transcript.d.ts +12 -0
- package/dist/evals/graders/transcript.js +57 -0
- package/dist/evals/graders/types.d.ts +20 -0
- package/dist/evals/graders/types.js +1 -0
- package/dist/evals/judgeProviders.d.ts +16 -0
- package/dist/evals/judgeProviders.js +27 -0
- package/dist/evals/lineDiff.d.ts +5 -0
- package/dist/evals/lineDiff.js +34 -0
- package/dist/evals/listStatus.d.ts +31 -0
- package/dist/evals/listStatus.js +48 -0
- package/dist/evals/manager.d.ts +187 -0
- package/dist/evals/manager.js +795 -0
- package/dist/evals/plan.d.ts +65 -0
- package/dist/evals/plan.js +621 -0
- package/dist/evals/planInternalAgent.d.ts +19 -0
- package/dist/evals/planInternalAgent.js +205 -0
- package/dist/evals/providerKeys.d.ts +1 -0
- package/dist/evals/providerKeys.js +21 -0
- package/dist/evals/read/agentTargets.d.ts +37 -0
- package/dist/evals/read/agentTargets.js +101 -0
- package/dist/evals/read/api.d.ts +208 -0
- package/dist/evals/read/api.js +363 -0
- package/dist/evals/read/compare.d.ts +4 -0
- package/dist/evals/read/compare.js +39 -0
- package/dist/evals/read/datasets.d.ts +105 -0
- package/dist/evals/read/datasets.js +318 -0
- package/dist/evals/read/list.d.ts +66 -0
- package/dist/evals/read/list.js +182 -0
- package/dist/evals/read/results.d.ts +46 -0
- package/dist/evals/read/results.js +107 -0
- package/dist/evals/read/runs.d.ts +154 -0
- package/dist/evals/read/runs.js +295 -0
- package/dist/evals/read/suggestions.d.ts +43 -0
- package/dist/evals/read/suggestions.js +73 -0
- package/dist/evals/runner/FakeAgentBackend.d.ts +40 -0
- package/dist/evals/runner/FakeAgentBackend.js +82 -0
- package/dist/evals/runner/agentCaseRunner.d.ts +24 -0
- package/dist/evals/runner/agentCaseRunner.js +388 -0
- package/dist/evals/runner/caseRunner.d.ts +8 -0
- package/dist/evals/runner/caseRunner.js +28 -0
- package/dist/evals/runner/conversationSubject.d.ts +18 -0
- package/dist/evals/runner/conversationSubject.js +123 -0
- package/dist/evals/runner/internalAgentCaseRunner.d.ts +6 -0
- package/dist/evals/runner/internalAgentCaseRunner.js +250 -0
- package/dist/evals/runner/simulatorModel.d.ts +51 -0
- package/dist/evals/runner/simulatorModel.js +58 -0
- package/dist/evals/runner/simulatorPrompt.d.ts +29 -0
- package/dist/evals/runner/simulatorPrompt.js +59 -0
- package/dist/evals/runner/stageCaseRunner.d.ts +30 -0
- package/dist/evals/runner/stageCaseRunner.js +279 -0
- package/dist/evals/runner/userDriver.d.ts +71 -0
- package/dist/evals/runner/userDriver.js +104 -0
- package/dist/evals/scoring.d.ts +34 -0
- package/dist/evals/scoring.js +67 -0
- package/dist/evals/snapshot.d.ts +191 -0
- package/dist/evals/snapshot.js +70 -0
- package/dist/evals/stageDiff.d.ts +8 -0
- package/dist/evals/stageDiff.js +48 -0
- package/dist/evals/stats.d.ts +95 -0
- package/dist/evals/stats.js +220 -0
- package/dist/evals/statsAdapter.d.ts +4 -0
- package/dist/evals/statsAdapter.js +30 -0
- package/dist/evals/statsTypes.d.ts +51 -0
- package/dist/evals/statsTypes.js +18 -0
- package/dist/evals/storeError.d.ts +5 -0
- package/dist/evals/storeError.js +8 -0
- package/dist/evals/storeTypes.d.ts +207 -0
- package/dist/evals/storeTypes.js +1 -0
- package/dist/evals/tTable.d.ts +3 -0
- package/dist/evals/tTable.js +15 -0
- package/dist/evals/triage/constants.d.ts +26 -0
- package/dist/evals/triage/constants.js +33 -0
- package/dist/evals/triage/credentials.d.ts +13 -0
- package/dist/evals/triage/credentials.js +20 -0
- package/dist/evals/triage/jevClassifier.d.ts +14 -0
- package/dist/evals/triage/jevClassifier.js +124 -0
- package/dist/evals/triage/judgeClassifier.d.ts +10 -0
- package/dist/evals/triage/judgeClassifier.js +76 -0
- package/dist/evals/triage/prompts.d.ts +16 -0
- package/dist/evals/triage/prompts.js +64 -0
- package/dist/evals/triage/service.d.ts +79 -0
- package/dist/evals/triage/service.js +217 -0
- package/dist/evals/triage/stateBuilder.d.ts +19 -0
- package/dist/evals/triage/stateBuilder.js +125 -0
- package/dist/evals/triage/types.d.ts +60 -0
- package/dist/evals/triage/types.js +8 -0
- package/dist/evals/types.d.ts +357 -0
- package/dist/evals/types.js +4 -0
- package/dist/guide/cardContract.d.ts +133 -0
- package/dist/guide/cardContract.js +156 -0
- package/dist/guide/chat.d.ts +45 -0
- package/dist/guide/chat.js +378 -0
- package/dist/guide/knowledge/index.d.ts +14 -0
- package/dist/guide/knowledge/index.js +109 -0
- package/dist/guide/knowledge/topics/agents.d.ts +2 -0
- package/dist/guide/knowledge/topics/agents.js +58 -0
- package/dist/guide/knowledge/topics/bestPractices.d.ts +2 -0
- package/dist/guide/knowledge/topics/bestPractices.js +67 -0
- package/dist/guide/knowledge/topics/connections.d.ts +2 -0
- package/dist/guide/knowledge/topics/connections.js +66 -0
- package/dist/guide/knowledge/topics/consoleMap.d.ts +2 -0
- package/dist/guide/knowledge/topics/consoleMap.js +55 -0
- package/dist/guide/knowledge/topics/evals.d.ts +2 -0
- package/dist/guide/knowledge/topics/evals.js +72 -0
- package/dist/guide/knowledge/topics/limitations.d.ts +2 -0
- package/dist/guide/knowledge/topics/limitations.js +84 -0
- package/dist/guide/knowledge/topics/mcpSkillsBrowser.d.ts +2 -0
- package/dist/guide/knowledge/topics/mcpSkillsBrowser.js +57 -0
- package/dist/guide/knowledge/topics/memories.d.ts +2 -0
- package/dist/guide/knowledge/topics/memories.js +63 -0
- package/dist/guide/knowledge/topics/modelsAndProviders.d.ts +2 -0
- package/dist/guide/knowledge/topics/modelsAndProviders.js +56 -0
- package/dist/guide/knowledge/topics/overview.d.ts +2 -0
- package/dist/guide/knowledge/topics/overview.js +56 -0
- package/dist/guide/knowledge/topics/pipelines.d.ts +2 -0
- package/dist/guide/knowledge/topics/pipelines.js +93 -0
- package/dist/guide/knowledge/topics/stages.d.ts +2 -0
- package/dist/guide/knowledge/topics/stages.js +54 -0
- package/dist/guide/knowledge/topics/tasks.d.ts +2 -0
- package/dist/guide/knowledge/topics/tasks.js +52 -0
- package/dist/guide/knowledge/topics/triggers.d.ts +2 -0
- package/dist/guide/knowledge/topics/triggers.js +68 -0
- package/dist/guide/knowledge/topics/troubleshooting.d.ts +2 -0
- package/dist/guide/knowledge/topics/troubleshooting.js +85 -0
- package/dist/guide/knowledgeTools.d.ts +2 -0
- package/dist/guide/knowledgeTools.js +56 -0
- package/dist/guide/showUi.d.ts +6 -0
- package/dist/guide/showUi.js +546 -0
- package/dist/guide/threadStore.d.ts +48 -0
- package/dist/guide/threadStore.js +189 -0
- package/dist/guide/writeRoot.d.ts +11 -0
- package/dist/guide/writeRoot.js +70 -0
- package/dist/hosted/translate.js +2 -0
- package/dist/mcp/agentTools.js +12 -0
- package/dist/mcp/catalogTools.js +18 -20
- package/dist/mcp/catalogWriteTools.d.ts +8 -0
- package/dist/mcp/catalogWriteTools.js +225 -0
- package/dist/mcp/connectionTools.d.ts +3 -0
- package/dist/mcp/connectionTools.js +85 -0
- package/dist/mcp/controlTools.js +34 -1
- package/dist/mcp/deps.d.ts +16 -0
- package/dist/mcp/evalTools.d.ts +3 -0
- package/dist/mcp/evalTools.js +943 -0
- package/dist/mcp/memoryHostTools.d.ts +3 -0
- package/dist/mcp/memoryHostTools.js +72 -0
- package/dist/mcp/pipelineAuthorTools.js +14 -1
- package/dist/mcp/server.js +6 -0
- package/dist/mcp/tools.js +8 -0
- package/dist/mcp/triggerTools.js +77 -31
- package/dist/models/response.d.ts +7 -2
- package/dist/models/response.js +6 -1
- package/dist/operatorAgent/agentAuthorContext.d.ts +1 -1
- package/dist/operatorAgent/agentAuthorContext.js +1 -1
- package/dist/operatorAgent/evalGate.d.ts +22 -0
- package/dist/operatorAgent/evalGate.js +77 -0
- package/dist/operatorAgent/fakeHost.js +10 -2
- package/dist/operatorAgent/host.js +18 -2
- package/dist/operatorAgent/index.d.ts +1 -0
- package/dist/operatorAgent/index.js +1 -0
- package/dist/operatorAgent/internalAgents.d.ts +11 -0
- package/dist/operatorAgent/internalAgents.js +13 -0
- package/dist/operatorAgent/piModel.d.ts +5 -1
- package/dist/operatorAgent/piModel.js +32 -14
- package/dist/operatorAgent/profiles/agentAuthor.d.ts +3 -1
- package/dist/operatorAgent/profiles/agentAuthor.js +201 -32
- package/dist/operatorAgent/profiles/agentAuthorPlaybook.d.ts +1 -1
- package/dist/operatorAgent/profiles/agentAuthorPlaybook.js +48 -15
- package/dist/operatorAgent/profiles/authorReadable.d.ts +2 -0
- package/dist/operatorAgent/profiles/authorReadable.js +26 -0
- package/dist/operatorAgent/profiles/guideConfirm.d.ts +21 -0
- package/dist/operatorAgent/profiles/guideConfirm.js +100 -0
- package/dist/operatorAgent/profiles/guideTools.d.ts +7 -0
- package/dist/operatorAgent/profiles/guideTools.js +123 -0
- package/dist/operatorAgent/profiles/stageflowGuide.d.ts +7 -0
- package/dist/operatorAgent/profiles/stageflowGuide.js +132 -0
- package/dist/operatorAgent/profiles/workshopAuthor.js +23 -0
- package/dist/operatorAgent/profiles/workshopAuthorPlaybook.d.ts +1 -1
- package/dist/operatorAgent/profiles/workshopAuthorPlaybook.js +9 -1
- package/dist/operatorAgent/profiles/workshopAuthorTools.d.ts +2 -0
- package/dist/operatorAgent/profiles/workshopAuthorTools.js +197 -9
- package/dist/operatorAgent/types.d.ts +23 -0
- package/dist/package-meta.d.ts +1 -1
- package/dist/package-meta.js +1 -1
- package/dist/runstore/port.d.ts +43 -0
- package/dist/runstore/retention.d.ts +2 -1
- package/dist/runstore/retention.js +3 -7
- package/dist/runstore/runStatusClock.d.ts +17 -0
- package/dist/runstore/runStatusClock.js +19 -0
- package/dist/runstore/sqlite/SqliteRunStore.d.ts +29 -0
- package/dist/runstore/sqlite/SqliteRunStore.js +144 -50
- package/dist/runstore/sqlite/evalStore.d.ts +20 -0
- package/dist/runstore/sqlite/evalStore.js +541 -0
- package/dist/runstore/sqlite/migrations/015-evals.d.ts +12 -0
- package/dist/runstore/sqlite/migrations/015-evals.js +157 -0
- package/dist/runstore/sqlite/migrations/016-eval-subject-ref.d.ts +12 -0
- package/dist/runstore/sqlite/migrations/016-eval-subject-ref.js +20 -0
- package/dist/runstore/sqlite/migrations/017-eval-result-suggestions.d.ts +11 -0
- package/dist/runstore/sqlite/migrations/017-eval-result-suggestions.js +36 -0
- package/dist/runstore/sqlite/migrations/index.js +6 -0
- package/dist/runstore/sqlite/suggestionStore.d.ts +14 -0
- package/dist/runstore/sqlite/suggestionStore.js +132 -0
- package/dist/runstore/workspaceLayout.js +6 -1
- package/dist/runtime/artifactDiskInspect.d.ts +14 -0
- package/dist/runtime/artifactDiskInspect.js +52 -0
- package/dist/runtime/artifactPath.d.ts +24 -0
- package/dist/runtime/artifactPath.js +84 -0
- package/dist/runtime/completionCheckRunner.js +15 -43
- package/dist/runtime/githubPollSource.d.ts +22 -4
- package/dist/runtime/githubPollSource.js +93 -8
- package/dist/runtime/pipelineRunner.d.ts +2 -1
- package/dist/runtime/pipelineRunner.js +2 -0
- package/dist/runtime/priorArtifactGrants.d.ts +8 -0
- package/dist/runtime/priorArtifactGrants.js +105 -0
- package/dist/runtime/runActionEligibility.d.ts +46 -0
- package/dist/runtime/runActionEligibility.js +123 -0
- package/dist/runtime/runManager.d.ts +6 -2
- package/dist/runtime/runManager.js +60 -16
- package/dist/runtime/runRetentionSweep.js +2 -2
- package/dist/runtime/runRetryCoordinator.d.ts +2 -14
- package/dist/runtime/runRetryCoordinator.js +3 -67
- package/dist/runtime/sandboxContainer.d.ts +7 -0
- package/dist/runtime/sandboxContainer.js +48 -6
- package/dist/runtime/sandboxDesk.d.ts +2 -1
- package/dist/runtime/sandboxDesk.js +11 -12
- package/dist/runtime/stageEnvironment.js +1 -0
- package/dist/runtime/stageRoots.d.ts +12 -0
- package/dist/runtime/stageSandbox.d.ts +2 -1
- package/dist/runtime/stageSandbox.js +18 -2
- package/dist/runtime/stageWorker.js +20 -0
- package/dist/server/agentAuthorRoutes.js +59 -1
- package/dist/server/bootstrap.d.ts +13 -0
- package/dist/server/bootstrap.js +69 -0
- package/dist/server/controlToken.js +3 -0
- package/dist/server/evalInternalRoutes.d.ts +14 -0
- package/dist/server/evalInternalRoutes.js +87 -0
- package/dist/server/evalRouteContext.d.ts +46 -0
- package/dist/server/evalRouteContext.js +43 -0
- package/dist/server/evalRoutes.d.ts +5 -0
- package/dist/server/evalRoutes.js +1456 -0
- package/dist/server/evalTriageRoutes.d.ts +2 -0
- package/dist/server/evalTriageRoutes.js +95 -0
- package/dist/server/guideRoutes.d.ts +5 -0
- package/dist/server/guideRoutes.js +166 -0
- package/dist/server/http.d.ts +15 -0
- package/dist/server/http.js +67 -3
- package/dist/server/internalAgentRoutes.d.ts +6 -0
- package/dist/server/internalAgentRoutes.js +12 -0
- package/dist/server/mcpHost.js +5 -0
- package/dist/toolkit/meta.d.ts +1 -1
- package/dist/toolkit/meta.js +53 -0
- package/dist/toolkit/profiles.d.ts +5 -0
- package/dist/toolkit/profiles.js +13 -1
- package/dist/tools/askOperator.js +2 -1
- package/dist/tools/emitStageEnvelope.d.ts +17 -16
- package/dist/tools/emitStageEnvelope.js +46 -32
- package/dist/tools/stageToolNames.d.ts +4 -0
- package/dist/tools/stageToolNames.js +4 -0
- package/dist/tools/workingState.d.ts +40 -0
- package/dist/tools/workingState.js +108 -0
- package/dist/tools/writeStageArtifact.d.ts +0 -9
- package/dist/tools/writeStageArtifact.js +2 -1
- package/dist/types/stageflowManifest.d.ts +6 -0
- package/dist/ui/assets/{MarkdownText-D7tYaIFO.js → MarkdownText-S5C_vemc.js} +2 -2
- package/dist/ui/assets/adapters-C0l4ruiR.js +17 -0
- package/dist/ui/assets/chat-CjnsmvqR.js +2 -0
- package/dist/ui/assets/chat-DFDd4UjS.css +2 -0
- package/dist/ui/assets/{main-BFo79krC.css → main-Del1GeLf.css} +1 -1
- package/dist/ui/assets/main-_yXVqcW9.js +277 -0
- package/dist/ui/chat.html +3 -4
- package/dist/ui/index.html +3 -4
- package/dist/workshop/chatTurn.js +1 -1
- package/dist/workshop/workshopPin.d.ts +3 -3
- package/dist/workshop/workshopPin.js +16 -15
- package/examples/README.md +2 -0
- package/examples/compaction-stress/README.md +30 -0
- package/examples/compaction-stress/check-run.mjs +53 -0
- package/examples/compaction-stress/collect-evidence.yaml +60 -0
- package/examples/compaction-stress/compaction-stress.pipeline.yaml +141 -0
- package/examples/compaction-stress/compaction-stress.task.yaml +15 -0
- package/examples/compaction-stress/generate-evidence.mjs +110 -0
- package/examples/compaction-stress/root-cause.yaml +26 -0
- package/examples/compaction-stress/stageflow.yaml +9 -0
- package/examples/compaction-stress/write-postmortem.yaml +40 -0
- package/examples/document-policy-review/README.md +35 -0
- package/examples/document-policy-review/agents/compliance.agent.yaml +42 -0
- package/examples/document-policy-review/documents/acme-vendor-sow.md +15 -0
- package/examples/document-policy-review/documents/support-runbook.md +13 -0
- package/examples/document-policy-review/policies/acceptable-use.md +13 -0
- package/examples/document-policy-review/policies/data-handling.md +17 -0
- package/examples/document-policy-review/policies/vendor-security.md +17 -0
- package/examples/evals/README.md +141 -0
- package/examples/evals/critique-sample.dataset.jsonl +8 -0
- package/examples/evals/critique-sample.eval.yaml +34 -0
- package/examples/evals/critique-sample.task.yaml +4 -0
- package/examples/evals/eval-sample.pipeline.yaml +35 -0
- package/examples/evals/guide-behavior.dataset.jsonl +15 -0
- package/examples/evals/guide-behavior.eval.yaml +35 -0
- package/examples/evals/guide-depth.dataset.jsonl +35 -0
- package/examples/evals/guide-depth.eval.yaml +169 -0
- package/examples/evals/support-sample.agent.yaml +20 -0
- package/examples/evals/support-sample.dataset.jsonl +6 -0
- package/examples/evals/support-sample.eval.yaml +38 -0
- package/examples/memory/evals/guide-behavior.dataset.jsonl +15 -0
- package/examples/memory/evals/guide-behavior.eval.yaml +35 -0
- package/examples/memory/evals/guide-depth.dataset.jsonl +35 -0
- package/examples/memory/evals/guide-depth.eval.yaml +169 -0
- package/examples/stageflow.yaml +6 -0
- package/package.json +1 -1
- package/skills/stageflow/SKILL.md +5 -2
- package/skills/stageflow/references/control-surface.md +4 -2
- package/skills/stageflow/references/evals.md +68 -0
- package/skills/stageflow-author/SKILL.md +4 -0
- package/skills/stageflow-run/SKILL.md +4 -0
- package/dist/ui/assets/AssistantRuntimeProvider-CsiPFhCT.js +0 -9
- package/dist/ui/assets/ReadonlyStore-BfXqHykm.js +0 -6
- package/dist/ui/assets/chat-BbLukq9W.js +0 -5
- package/dist/ui/assets/chat-zIZU_bP1.css +0 -2
- package/dist/ui/assets/main-CijYMeM0.js +0 -259
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import { PACKAGE_VERSION } from "../../../package-meta.js";
|
|
2
|
+
/**
|
|
3
|
+
* Eval runs, their cases, results, grades and the author's reviews, plus `runs.origin` so stage runs
|
|
4
|
+
* started by an eval can be hidden from normal run views. `eval_run_counters` keeps `run_number`
|
|
5
|
+
* from being reused after an eval run is deleted.
|
|
6
|
+
*/
|
|
7
|
+
export const MIGRATION_015 = {
|
|
8
|
+
version: 15,
|
|
9
|
+
name: "015_evals",
|
|
10
|
+
minStageflowVersion: PACKAGE_VERSION,
|
|
11
|
+
up(db) {
|
|
12
|
+
db.exec(`
|
|
13
|
+
ALTER TABLE runs ADD COLUMN origin TEXT;
|
|
14
|
+
CREATE INDEX runs_origin ON runs (origin);
|
|
15
|
+
|
|
16
|
+
CREATE TABLE eval_runs (
|
|
17
|
+
eval_run_id TEXT PRIMARY KEY,
|
|
18
|
+
project_root TEXT NOT NULL,
|
|
19
|
+
eval_id TEXT NOT NULL,
|
|
20
|
+
run_number INTEGER NOT NULL,
|
|
21
|
+
eval_path TEXT NOT NULL,
|
|
22
|
+
eval_hash TEXT NOT NULL,
|
|
23
|
+
target_kind TEXT NOT NULL CHECK (target_kind IN ('stage','agent')),
|
|
24
|
+
target_ref_json TEXT NOT NULL,
|
|
25
|
+
target_version_hash TEXT NOT NULL,
|
|
26
|
+
dataset_path TEXT NOT NULL,
|
|
27
|
+
dataset_hash TEXT NOT NULL,
|
|
28
|
+
config_json TEXT NOT NULL,
|
|
29
|
+
config_fingerprint TEXT NOT NULL,
|
|
30
|
+
config_components_json TEXT NOT NULL,
|
|
31
|
+
status TEXT NOT NULL CHECK (status IN ('queued','running','completed','failed','cancelled','budget_stopped')),
|
|
32
|
+
status_reason TEXT,
|
|
33
|
+
passed INTEGER,
|
|
34
|
+
repeats INTEGER NOT NULL,
|
|
35
|
+
pass_threshold REAL NOT NULL,
|
|
36
|
+
max_cost_usd REAL NOT NULL,
|
|
37
|
+
cases_total INTEGER NOT NULL,
|
|
38
|
+
results_total INTEGER NOT NULL,
|
|
39
|
+
results_done INTEGER NOT NULL DEFAULT 0,
|
|
40
|
+
cost_usd REAL NOT NULL DEFAULT 0,
|
|
41
|
+
engine_name TEXT NOT NULL,
|
|
42
|
+
engine_version TEXT NOT NULL,
|
|
43
|
+
stageflow_version TEXT NOT NULL,
|
|
44
|
+
label TEXT,
|
|
45
|
+
started_via TEXT NOT NULL CHECK (started_via IN ('console','cli','mcp')),
|
|
46
|
+
caller_id TEXT,
|
|
47
|
+
summary_json TEXT,
|
|
48
|
+
created_at TEXT NOT NULL,
|
|
49
|
+
started_at TEXT,
|
|
50
|
+
finished_at TEXT
|
|
51
|
+
);
|
|
52
|
+
CREATE UNIQUE INDEX eval_runs_number ON eval_runs (project_root, eval_id, run_number);
|
|
53
|
+
CREATE INDEX eval_runs_eval ON eval_runs (project_root, eval_id, created_at DESC);
|
|
54
|
+
CREATE INDEX eval_runs_status ON eval_runs (status);
|
|
55
|
+
|
|
56
|
+
CREATE TABLE eval_run_counters (
|
|
57
|
+
project_root TEXT NOT NULL,
|
|
58
|
+
eval_id TEXT NOT NULL,
|
|
59
|
+
last_number INTEGER NOT NULL,
|
|
60
|
+
PRIMARY KEY (project_root, eval_id)
|
|
61
|
+
);
|
|
62
|
+
|
|
63
|
+
CREATE TABLE eval_cases (
|
|
64
|
+
eval_run_id TEXT NOT NULL REFERENCES eval_runs (eval_run_id) ON DELETE CASCADE,
|
|
65
|
+
case_id TEXT NOT NULL,
|
|
66
|
+
position INTEGER NOT NULL,
|
|
67
|
+
case_json TEXT NOT NULL,
|
|
68
|
+
PRIMARY KEY (eval_run_id, case_id)
|
|
69
|
+
);
|
|
70
|
+
|
|
71
|
+
CREATE TABLE eval_results (
|
|
72
|
+
eval_run_id TEXT NOT NULL REFERENCES eval_runs (eval_run_id) ON DELETE CASCADE,
|
|
73
|
+
case_id TEXT NOT NULL,
|
|
74
|
+
repeat_index INTEGER NOT NULL,
|
|
75
|
+
status TEXT NOT NULL CHECK (status IN ('completed','stage_failed','needs_input','error','skipped_budget','cancelled')),
|
|
76
|
+
error_code TEXT,
|
|
77
|
+
error_message TEXT,
|
|
78
|
+
stage_run_id TEXT,
|
|
79
|
+
output_json TEXT,
|
|
80
|
+
latency_ms INTEGER,
|
|
81
|
+
cost_usd REAL,
|
|
82
|
+
tokens_in INTEGER,
|
|
83
|
+
tokens_out INTEGER,
|
|
84
|
+
score REAL,
|
|
85
|
+
passed INTEGER NOT NULL DEFAULT 0,
|
|
86
|
+
has_grader_errors INTEGER NOT NULL DEFAULT 0,
|
|
87
|
+
budget_warnings_json TEXT,
|
|
88
|
+
started_at TEXT,
|
|
89
|
+
finished_at TEXT,
|
|
90
|
+
PRIMARY KEY (eval_run_id, case_id, repeat_index),
|
|
91
|
+
FOREIGN KEY (eval_run_id, case_id) REFERENCES eval_cases (eval_run_id, case_id) ON DELETE CASCADE
|
|
92
|
+
);
|
|
93
|
+
CREATE INDEX eval_results_stage_run ON eval_results (stage_run_id);
|
|
94
|
+
|
|
95
|
+
CREATE TABLE eval_grades (
|
|
96
|
+
eval_run_id TEXT NOT NULL,
|
|
97
|
+
case_id TEXT NOT NULL,
|
|
98
|
+
repeat_index INTEGER NOT NULL,
|
|
99
|
+
grader_id TEXT NOT NULL,
|
|
100
|
+
grader_type TEXT NOT NULL,
|
|
101
|
+
status TEXT NOT NULL CHECK (status IN ('graded','error')),
|
|
102
|
+
passed INTEGER,
|
|
103
|
+
score REAL,
|
|
104
|
+
raw_value REAL,
|
|
105
|
+
reason TEXT,
|
|
106
|
+
details_json TEXT,
|
|
107
|
+
error_code TEXT,
|
|
108
|
+
judge_model TEXT,
|
|
109
|
+
judge_cost_usd REAL,
|
|
110
|
+
judge_tokens_in INTEGER,
|
|
111
|
+
judge_tokens_out INTEGER,
|
|
112
|
+
PRIMARY KEY (eval_run_id, case_id, repeat_index, grader_id),
|
|
113
|
+
FOREIGN KEY (eval_run_id, case_id, repeat_index) REFERENCES eval_results (eval_run_id, case_id, repeat_index) ON DELETE CASCADE
|
|
114
|
+
);
|
|
115
|
+
|
|
116
|
+
CREATE TABLE eval_grader_versions (
|
|
117
|
+
project_root TEXT NOT NULL,
|
|
118
|
+
eval_id TEXT NOT NULL,
|
|
119
|
+
grader_id TEXT NOT NULL,
|
|
120
|
+
grader_hash TEXT NOT NULL,
|
|
121
|
+
ordinal INTEGER NOT NULL,
|
|
122
|
+
first_seen_at TEXT NOT NULL,
|
|
123
|
+
PRIMARY KEY (project_root, eval_id, grader_id, grader_hash)
|
|
124
|
+
);
|
|
125
|
+
|
|
126
|
+
CREATE TABLE eval_failure_types (
|
|
127
|
+
project_root TEXT NOT NULL,
|
|
128
|
+
eval_id TEXT NOT NULL,
|
|
129
|
+
name TEXT NOT NULL,
|
|
130
|
+
created_at TEXT NOT NULL,
|
|
131
|
+
PRIMARY KEY (project_root, eval_id, name COLLATE NOCASE)
|
|
132
|
+
);
|
|
133
|
+
|
|
134
|
+
CREATE TABLE eval_result_reviews (
|
|
135
|
+
eval_run_id TEXT NOT NULL,
|
|
136
|
+
case_id TEXT NOT NULL,
|
|
137
|
+
repeat_index INTEGER NOT NULL,
|
|
138
|
+
label TEXT NOT NULL CHECK (label IN ('pass','fail')),
|
|
139
|
+
note TEXT,
|
|
140
|
+
created_at TEXT NOT NULL,
|
|
141
|
+
updated_at TEXT NOT NULL,
|
|
142
|
+
PRIMARY KEY (eval_run_id, case_id, repeat_index),
|
|
143
|
+
FOREIGN KEY (eval_run_id, case_id, repeat_index) REFERENCES eval_results (eval_run_id, case_id, repeat_index) ON DELETE CASCADE
|
|
144
|
+
);
|
|
145
|
+
|
|
146
|
+
CREATE TABLE eval_result_tags (
|
|
147
|
+
eval_run_id TEXT NOT NULL,
|
|
148
|
+
case_id TEXT NOT NULL,
|
|
149
|
+
repeat_index INTEGER NOT NULL,
|
|
150
|
+
tag TEXT NOT NULL,
|
|
151
|
+
created_at TEXT NOT NULL,
|
|
152
|
+
PRIMARY KEY (eval_run_id, case_id, repeat_index, tag COLLATE NOCASE),
|
|
153
|
+
FOREIGN KEY (eval_run_id, case_id, repeat_index) REFERENCES eval_results (eval_run_id, case_id, repeat_index) ON DELETE CASCADE
|
|
154
|
+
);
|
|
155
|
+
`);
|
|
156
|
+
},
|
|
157
|
+
};
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type Database from "better-sqlite3";
|
|
2
|
+
/**
|
|
3
|
+
* Agent eval results point at an agent session, not a stage run: `subject_ref` is the root subject id
|
|
4
|
+
* (stage run id for stage results, agent session id for agent results). `turns` and `tool_call_count`
|
|
5
|
+
* are agent summaries. Additive; stage rows keep `stage_run_id` and are backfilled.
|
|
6
|
+
*/
|
|
7
|
+
export declare const MIGRATION_016: {
|
|
8
|
+
version: number;
|
|
9
|
+
name: string;
|
|
10
|
+
minStageflowVersion: string;
|
|
11
|
+
up(db: Database.Database): void;
|
|
12
|
+
};
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { PACKAGE_VERSION } from "../../../package-meta.js";
|
|
2
|
+
/**
|
|
3
|
+
* Agent eval results point at an agent session, not a stage run: `subject_ref` is the root subject id
|
|
4
|
+
* (stage run id for stage results, agent session id for agent results). `turns` and `tool_call_count`
|
|
5
|
+
* are agent summaries. Additive; stage rows keep `stage_run_id` and are backfilled.
|
|
6
|
+
*/
|
|
7
|
+
export const MIGRATION_016 = {
|
|
8
|
+
version: 16,
|
|
9
|
+
name: "016_eval_subject_ref",
|
|
10
|
+
minStageflowVersion: PACKAGE_VERSION,
|
|
11
|
+
up(db) {
|
|
12
|
+
db.exec(`
|
|
13
|
+
ALTER TABLE eval_results ADD COLUMN subject_ref TEXT;
|
|
14
|
+
ALTER TABLE eval_results ADD COLUMN turns INTEGER;
|
|
15
|
+
ALTER TABLE eval_results ADD COLUMN tool_call_count INTEGER;
|
|
16
|
+
UPDATE eval_results SET subject_ref = stage_run_id WHERE stage_run_id IS NOT NULL;
|
|
17
|
+
CREATE INDEX eval_results_subject_ref ON eval_results (subject_ref);
|
|
18
|
+
`);
|
|
19
|
+
},
|
|
20
|
+
};
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type Database from "better-sqlite3";
|
|
2
|
+
/**
|
|
3
|
+
* Suggested bucket and failure type per failed eval result, produced on request by a classifier
|
|
4
|
+
* (`jev` or `judge`). A suggestion becomes a tag only when the author accepts it. Additive.
|
|
5
|
+
*/
|
|
6
|
+
export declare const MIGRATION_017: {
|
|
7
|
+
version: number;
|
|
8
|
+
name: string;
|
|
9
|
+
minStageflowVersion: string;
|
|
10
|
+
up(db: Database.Database): void;
|
|
11
|
+
};
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { PACKAGE_VERSION } from "../../../package-meta.js";
|
|
2
|
+
/**
|
|
3
|
+
* Suggested bucket and failure type per failed eval result, produced on request by a classifier
|
|
4
|
+
* (`jev` or `judge`). A suggestion becomes a tag only when the author accepts it. Additive.
|
|
5
|
+
*/
|
|
6
|
+
export const MIGRATION_017 = {
|
|
7
|
+
version: 17,
|
|
8
|
+
name: "017_eval_result_suggestions",
|
|
9
|
+
minStageflowVersion: PACKAGE_VERSION,
|
|
10
|
+
up(db) {
|
|
11
|
+
db.exec(`
|
|
12
|
+
CREATE TABLE eval_result_suggestions (
|
|
13
|
+
id TEXT PRIMARY KEY,
|
|
14
|
+
eval_run_id TEXT NOT NULL,
|
|
15
|
+
case_id TEXT NOT NULL,
|
|
16
|
+
repeat_index INTEGER NOT NULL,
|
|
17
|
+
source TEXT NOT NULL CHECK (source IN ('jev','judge')),
|
|
18
|
+
bucket TEXT NOT NULL CHECK (bucket IN ('agent_wrong','grader_wrong','case_wrong','unclear')),
|
|
19
|
+
failure_type TEXT,
|
|
20
|
+
confidence REAL NOT NULL,
|
|
21
|
+
probabilities_json TEXT NOT NULL,
|
|
22
|
+
grader_false_alarms_json TEXT NOT NULL,
|
|
23
|
+
model TEXT NOT NULL,
|
|
24
|
+
prompt_version INTEGER NOT NULL,
|
|
25
|
+
status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','accepted','dismissed')),
|
|
26
|
+
batch_id TEXT NOT NULL,
|
|
27
|
+
cost_usd REAL NOT NULL DEFAULT 0,
|
|
28
|
+
state_truncated INTEGER NOT NULL DEFAULT 0,
|
|
29
|
+
created_at TEXT NOT NULL,
|
|
30
|
+
UNIQUE (eval_run_id, case_id, repeat_index, source),
|
|
31
|
+
FOREIGN KEY (eval_run_id, case_id, repeat_index) REFERENCES eval_results (eval_run_id, case_id, repeat_index) ON DELETE CASCADE
|
|
32
|
+
);
|
|
33
|
+
CREATE INDEX eval_result_suggestions_run ON eval_result_suggestions (eval_run_id, status);
|
|
34
|
+
`);
|
|
35
|
+
},
|
|
36
|
+
};
|
|
@@ -13,6 +13,9 @@ import { MIGRATION_011 } from "./011-email-dispatch-key.js";
|
|
|
13
13
|
import { MIGRATION_012 } from "./012-trigger-fires.js";
|
|
14
14
|
import { MIGRATION_013 } from "./013-agents.js";
|
|
15
15
|
import { MIGRATION_014 } from "./014-agent-deliveries.js";
|
|
16
|
+
import { MIGRATION_015 } from "./015-evals.js";
|
|
17
|
+
import { MIGRATION_016 } from "./016-eval-subject-ref.js";
|
|
18
|
+
import { MIGRATION_017 } from "./017-eval-result-suggestions.js";
|
|
16
19
|
const DEFAULT_MIGRATIONS = [
|
|
17
20
|
MIGRATION_001,
|
|
18
21
|
MIGRATION_002,
|
|
@@ -28,6 +31,9 @@ const DEFAULT_MIGRATIONS = [
|
|
|
28
31
|
MIGRATION_012,
|
|
29
32
|
MIGRATION_013,
|
|
30
33
|
MIGRATION_014,
|
|
34
|
+
MIGRATION_015,
|
|
35
|
+
MIGRATION_016,
|
|
36
|
+
MIGRATION_017,
|
|
31
37
|
];
|
|
32
38
|
export const CURRENT_SCHEMA_VERSION = DEFAULT_MIGRATIONS[DEFAULT_MIGRATIONS.length - 1].version;
|
|
33
39
|
function throwStoreSchemaTooNew(db, version, maxKnown) {
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type Database from "better-sqlite3";
|
|
2
|
+
import type { SuggestionInput, SuggestionRow } from "../../evals/triage/types.js";
|
|
3
|
+
/** Inserts or replaces pending suggestions; accepted and dismissed ones are kept. Returns the rows this call wrote. */
|
|
4
|
+
export declare function upsertEvalSuggestions(db: Database.Database, evalRunId: string, inputs: SuggestionInput[]): SuggestionRow[];
|
|
5
|
+
export declare function listEvalSuggestions(db: Database.Database, evalRunId: string, status?: SuggestionRow["status"]): SuggestionRow[];
|
|
6
|
+
export declare function dismissEvalSuggestions(db: Database.Database, evalRunId: string, ids: string[]): number;
|
|
7
|
+
/**
|
|
8
|
+
* Marks pending suggestions accepted and adds their failure type and `bucket:<bucket>` tags to the result
|
|
9
|
+
* through `putEvalReview`, keeping tags already there. Returns how many were accepted and how many results gained a tag.
|
|
10
|
+
*/
|
|
11
|
+
export declare function acceptEvalSuggestions(db: Database.Database, evalRunId: string, ids: string[]): {
|
|
12
|
+
accepted: number;
|
|
13
|
+
tagged: number;
|
|
14
|
+
};
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import { randomBytes } from "node:crypto";
|
|
2
|
+
import { EvalStoreError } from "../../evals/storeError.js";
|
|
3
|
+
import { BUCKET_TAG_PREFIX } from "../../evals/triage/constants.js";
|
|
4
|
+
import { getEvalReviews, putEvalReview } from "./evalStore.js";
|
|
5
|
+
const now = () => new Date().toISOString();
|
|
6
|
+
function suggestionFromDb(row) {
|
|
7
|
+
return {
|
|
8
|
+
id: row.id,
|
|
9
|
+
evalRunId: row.eval_run_id,
|
|
10
|
+
caseId: row.case_id,
|
|
11
|
+
repeatIndex: row.repeat_index,
|
|
12
|
+
source: row.source,
|
|
13
|
+
bucket: row.bucket,
|
|
14
|
+
failureType: row.failure_type,
|
|
15
|
+
confidence: row.confidence,
|
|
16
|
+
bucketProbabilities: JSON.parse(row.probabilities_json),
|
|
17
|
+
graderFalseAlarms: JSON.parse(row.grader_false_alarms_json),
|
|
18
|
+
model: row.model,
|
|
19
|
+
promptVersion: row.prompt_version,
|
|
20
|
+
status: row.status,
|
|
21
|
+
batchId: row.batch_id,
|
|
22
|
+
costUsd: row.cost_usd,
|
|
23
|
+
stateTruncated: row.state_truncated === 1,
|
|
24
|
+
createdAt: row.created_at,
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
/** Inserts or replaces pending suggestions; accepted and dismissed ones are kept. Returns the rows this call wrote. */
|
|
28
|
+
export function upsertEvalSuggestions(db, evalRunId, inputs) {
|
|
29
|
+
const written = [];
|
|
30
|
+
const at = now();
|
|
31
|
+
db.transaction(() => {
|
|
32
|
+
for (const input of inputs) {
|
|
33
|
+
const info = db
|
|
34
|
+
.prepare(`INSERT INTO eval_result_suggestions
|
|
35
|
+
(id, eval_run_id, case_id, repeat_index, source, bucket, failure_type, confidence, probabilities_json,
|
|
36
|
+
grader_false_alarms_json, model, prompt_version, status, batch_id, cost_usd, state_truncated, created_at)
|
|
37
|
+
VALUES (@id, @eval_run_id, @case_id, @repeat_index, @source, @bucket, @failure_type, @confidence, @probabilities_json,
|
|
38
|
+
@grader_false_alarms_json, @model, @prompt_version, 'pending', @batch_id, @cost_usd, @state_truncated, @at)
|
|
39
|
+
ON CONFLICT (eval_run_id, case_id, repeat_index, source) DO UPDATE SET
|
|
40
|
+
bucket = excluded.bucket, failure_type = excluded.failure_type, confidence = excluded.confidence,
|
|
41
|
+
probabilities_json = excluded.probabilities_json, grader_false_alarms_json = excluded.grader_false_alarms_json,
|
|
42
|
+
model = excluded.model, prompt_version = excluded.prompt_version, batch_id = excluded.batch_id,
|
|
43
|
+
cost_usd = excluded.cost_usd, state_truncated = excluded.state_truncated, created_at = excluded.created_at
|
|
44
|
+
WHERE eval_result_suggestions.status = 'pending'`)
|
|
45
|
+
.run({
|
|
46
|
+
id: `sg_${randomBytes(8).toString("hex")}`,
|
|
47
|
+
eval_run_id: evalRunId,
|
|
48
|
+
case_id: input.caseId,
|
|
49
|
+
repeat_index: input.repeatIndex,
|
|
50
|
+
source: input.source,
|
|
51
|
+
bucket: input.bucket,
|
|
52
|
+
failure_type: input.failureType,
|
|
53
|
+
confidence: input.confidence,
|
|
54
|
+
probabilities_json: JSON.stringify(input.bucketProbabilities),
|
|
55
|
+
grader_false_alarms_json: JSON.stringify(input.graderFalseAlarms),
|
|
56
|
+
model: input.model,
|
|
57
|
+
prompt_version: input.promptVersion,
|
|
58
|
+
batch_id: input.batchId,
|
|
59
|
+
cost_usd: input.costUsd,
|
|
60
|
+
state_truncated: input.stateTruncated ? 1 : 0,
|
|
61
|
+
at,
|
|
62
|
+
});
|
|
63
|
+
if (info.changes === 0)
|
|
64
|
+
continue;
|
|
65
|
+
const row = db
|
|
66
|
+
.prepare(`SELECT * FROM eval_result_suggestions WHERE eval_run_id = ? AND case_id = ? AND repeat_index = ? AND source = ?`)
|
|
67
|
+
.get(evalRunId, input.caseId, input.repeatIndex, input.source);
|
|
68
|
+
written.push(suggestionFromDb(row));
|
|
69
|
+
}
|
|
70
|
+
})();
|
|
71
|
+
return written;
|
|
72
|
+
}
|
|
73
|
+
export function listEvalSuggestions(db, evalRunId, status) {
|
|
74
|
+
const rows = (status !== undefined
|
|
75
|
+
? db
|
|
76
|
+
.prepare(`SELECT * FROM eval_result_suggestions WHERE eval_run_id = ? AND status = ? ORDER BY case_id, repeat_index, source`)
|
|
77
|
+
.all(evalRunId, status)
|
|
78
|
+
: db
|
|
79
|
+
.prepare(`SELECT * FROM eval_result_suggestions WHERE eval_run_id = ? ORDER BY case_id, repeat_index, source`)
|
|
80
|
+
.all(evalRunId));
|
|
81
|
+
return rows.map(suggestionFromDb);
|
|
82
|
+
}
|
|
83
|
+
export function dismissEvalSuggestions(db, evalRunId, ids) {
|
|
84
|
+
let changed = 0;
|
|
85
|
+
db.transaction(() => {
|
|
86
|
+
for (const id of ids) {
|
|
87
|
+
changed += db
|
|
88
|
+
.prepare(`UPDATE eval_result_suggestions SET status = 'dismissed' WHERE eval_run_id = ? AND id = ? AND status = 'pending'`)
|
|
89
|
+
.run(evalRunId, id).changes;
|
|
90
|
+
}
|
|
91
|
+
})();
|
|
92
|
+
return changed;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Marks pending suggestions accepted and adds their failure type and `bucket:<bucket>` tags to the result
|
|
96
|
+
* through `putEvalReview`, keeping tags already there. Returns how many were accepted and how many results gained a tag.
|
|
97
|
+
*/
|
|
98
|
+
export function acceptEvalSuggestions(db, evalRunId, ids) {
|
|
99
|
+
let accepted = 0;
|
|
100
|
+
let tagged = 0;
|
|
101
|
+
db.transaction(() => {
|
|
102
|
+
for (const id of ids) {
|
|
103
|
+
const found = db
|
|
104
|
+
.prepare(`SELECT * FROM eval_result_suggestions WHERE eval_run_id = ? AND id = ? AND status = 'pending'`)
|
|
105
|
+
.get(evalRunId, id);
|
|
106
|
+
if (found === undefined)
|
|
107
|
+
continue;
|
|
108
|
+
const suggestion = suggestionFromDb(found);
|
|
109
|
+
const add = [
|
|
110
|
+
...(suggestion.failureType !== null ? [suggestion.failureType] : []),
|
|
111
|
+
`${BUCKET_TAG_PREFIX}${suggestion.bucket}`,
|
|
112
|
+
];
|
|
113
|
+
const have = getEvalReviews(db, evalRunId).find((r) => r.caseId === suggestion.caseId && r.repeatIndex === suggestion.repeatIndex)?.tags ?? [];
|
|
114
|
+
const haveKeys = new Set(have.map((t) => t.toLowerCase()));
|
|
115
|
+
const fresh = add.filter((t) => !haveKeys.has(t.toLowerCase()));
|
|
116
|
+
if (fresh.length > 0) {
|
|
117
|
+
try {
|
|
118
|
+
putEvalReview(db, evalRunId, suggestion.caseId, suggestion.repeatIndex, { tags: [...have, ...fresh] });
|
|
119
|
+
tagged += 1;
|
|
120
|
+
}
|
|
121
|
+
catch (err) {
|
|
122
|
+
if (err instanceof EvalStoreError && err.code === "EVAL_TAGS_ON_PASSED_RESULT")
|
|
123
|
+
continue;
|
|
124
|
+
throw err;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
db.prepare(`UPDATE eval_result_suggestions SET status = 'accepted' WHERE id = ?`).run(id);
|
|
128
|
+
accepted += 1;
|
|
129
|
+
}
|
|
130
|
+
})();
|
|
131
|
+
return { accepted, tagged };
|
|
132
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { access, lstat, mkdir, readdir, realpath } from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { globalCredentialRoot } from "../project/globalHome.js";
|
|
4
|
+
import { buildArtifactListedPath } from "../runtime/artifactPath.js";
|
|
4
5
|
import { STAGEFLOW_AGENT_AUTH_PATH_ENV } from "../runtime/credentialBinding.js";
|
|
5
6
|
import { SF_STAGE_WORKER } from "../runtime/stageWorkerProtocol.js";
|
|
6
7
|
export { stageDir } from "./paths.js";
|
|
@@ -186,7 +187,11 @@ export function resolveArtifactTarget(runWorkspaceDir, stageId, attempt, relativ
|
|
|
186
187
|
path.isAbsolute(relativeToArtifacts)) {
|
|
187
188
|
throw new Error("path escapes the stage artifacts directory");
|
|
188
189
|
}
|
|
189
|
-
const runRelativePath =
|
|
190
|
+
const runRelativePath = buildArtifactListedPath({
|
|
191
|
+
stageId,
|
|
192
|
+
attempt,
|
|
193
|
+
relative: relativeToArtifacts.split(path.sep).join("/"),
|
|
194
|
+
});
|
|
190
195
|
return {
|
|
191
196
|
absolutePath: resolved,
|
|
192
197
|
runRelativePath,
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export type ArtifactDiskFileType = "file" | "directory" | "symlink" | "other" | "missing";
|
|
2
|
+
export type ArtifactDiskPresence = {
|
|
3
|
+
file_type: ArtifactDiskFileType;
|
|
4
|
+
/** Lexical absolute path under root when the walk reached a leaf. */
|
|
5
|
+
absolutePath?: string;
|
|
6
|
+
/** From lstat when `file_type` is `file`. */
|
|
7
|
+
size_bytes?: number;
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* Closed-posture walk of `relativePath` under `rootDir`: lstat every segment,
|
|
11
|
+
* classify symlink / missing / file / directory / other. Does not follow
|
|
12
|
+
* symlinks. Escape attempts are `missing`. Non-ENOENT errors propagate.
|
|
13
|
+
*/
|
|
14
|
+
export declare function inspectArtifactUnderRoot(rootDir: string, relativePath: string): Promise<ArtifactDiskPresence>;
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { lstat } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
/**
|
|
4
|
+
* Closed-posture walk of `relativePath` under `rootDir`: lstat every segment,
|
|
5
|
+
* classify symlink / missing / file / directory / other. Does not follow
|
|
6
|
+
* symlinks. Escape attempts are `missing`. Non-ENOENT errors propagate.
|
|
7
|
+
*/
|
|
8
|
+
export async function inspectArtifactUnderRoot(rootDir, relativePath) {
|
|
9
|
+
const target = path.resolve(rootDir, relativePath);
|
|
10
|
+
const relative = path.relative(rootDir, target);
|
|
11
|
+
if (relative === "" ||
|
|
12
|
+
relative === ".." ||
|
|
13
|
+
relative.startsWith(`..${path.sep}`) ||
|
|
14
|
+
path.isAbsolute(relative)) {
|
|
15
|
+
return { file_type: "missing" };
|
|
16
|
+
}
|
|
17
|
+
try {
|
|
18
|
+
const segments = relative.split(path.sep).filter((segment) => segment !== "");
|
|
19
|
+
let current = rootDir;
|
|
20
|
+
let stats;
|
|
21
|
+
for (const segment of segments) {
|
|
22
|
+
current = path.join(current, segment);
|
|
23
|
+
stats = await lstat(current);
|
|
24
|
+
if (stats.isSymbolicLink()) {
|
|
25
|
+
return { file_type: "symlink", absolutePath: current };
|
|
26
|
+
}
|
|
27
|
+
if (current !== target && !stats.isDirectory()) {
|
|
28
|
+
return { file_type: "other", absolutePath: current };
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
if (stats === undefined) {
|
|
32
|
+
return { file_type: "missing" };
|
|
33
|
+
}
|
|
34
|
+
if (stats.isDirectory()) {
|
|
35
|
+
return { file_type: "directory", absolutePath: current };
|
|
36
|
+
}
|
|
37
|
+
if (!stats.isFile()) {
|
|
38
|
+
return { file_type: "other", absolutePath: current };
|
|
39
|
+
}
|
|
40
|
+
return {
|
|
41
|
+
file_type: "file",
|
|
42
|
+
absolutePath: current,
|
|
43
|
+
size_bytes: stats.size,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
catch (error) {
|
|
47
|
+
if (error.code === "ENOENT") {
|
|
48
|
+
return { file_type: "missing" };
|
|
49
|
+
}
|
|
50
|
+
throw error;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/** Parts of a run-relative artifact path under `stages/.../artifacts/`. */
|
|
2
|
+
export type ArtifactPathParts = {
|
|
3
|
+
stageId: string;
|
|
4
|
+
attempt: number;
|
|
5
|
+
relative: string;
|
|
6
|
+
};
|
|
7
|
+
export type ArtifactPathMode = {
|
|
8
|
+
kind: "grantable";
|
|
9
|
+
} | {
|
|
10
|
+
kind: "current_attempt";
|
|
11
|
+
stageId: string;
|
|
12
|
+
attempt: number;
|
|
13
|
+
};
|
|
14
|
+
export declare function attemptArtifactsPrefix(stageId: string, attempt: number): string;
|
|
15
|
+
export declare function buildArtifactListedPath(parts: ArtifactPathParts): string;
|
|
16
|
+
/**
|
|
17
|
+
* Accept a listed artifact path under a mode.
|
|
18
|
+
* - `grantable`: full run-relative path to a prior stage/attempt artifact file.
|
|
19
|
+
* - `current_attempt`: that attempt's run-relative prefix, or a bare path under
|
|
20
|
+
* its artifacts dir; foreign `stages/...` paths are rejected.
|
|
21
|
+
*/
|
|
22
|
+
export declare function acceptArtifactListedPath(listedPath: string, mode: ArtifactPathMode): ArtifactPathParts | undefined;
|
|
23
|
+
export declare function parseArtifactListedPath(listedPath: string): ArtifactPathParts | undefined;
|
|
24
|
+
export declare function isGrantableArtifactPath(listedPath: string): boolean;
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
export function attemptArtifactsPrefix(stageId, attempt) {
|
|
3
|
+
return path.posix.join("stages", stageId, "attempts", String(attempt), "artifacts");
|
|
4
|
+
}
|
|
5
|
+
export function buildArtifactListedPath(parts) {
|
|
6
|
+
const relative = parts.relative.replace(/\\/g, "/");
|
|
7
|
+
const segments = relative.split("/").filter((segment) => segment.length > 0);
|
|
8
|
+
return path.posix.join("stages", parts.stageId, "attempts", String(parts.attempt), "artifacts", ...segments);
|
|
9
|
+
}
|
|
10
|
+
function parseStrictGrantable(listedPath) {
|
|
11
|
+
if (typeof listedPath !== "string" || listedPath.length === 0)
|
|
12
|
+
return undefined;
|
|
13
|
+
const posix = listedPath.replace(/\\/g, "/");
|
|
14
|
+
if (path.isAbsolute(listedPath) || path.posix.isAbsolute(posix))
|
|
15
|
+
return undefined;
|
|
16
|
+
if (posix.includes("\0") || posix.includes(".."))
|
|
17
|
+
return undefined;
|
|
18
|
+
const parts = posix.split("/").filter((part) => part.length > 0);
|
|
19
|
+
if (parts.length < 6)
|
|
20
|
+
return undefined;
|
|
21
|
+
if (parts[0] !== "stages" || parts[2] !== "attempts" || parts[4] !== "artifacts") {
|
|
22
|
+
return undefined;
|
|
23
|
+
}
|
|
24
|
+
const attemptRaw = parts[3];
|
|
25
|
+
if (!/^\d+$/.test(attemptRaw))
|
|
26
|
+
return undefined;
|
|
27
|
+
const attempt = Number(attemptRaw);
|
|
28
|
+
if (!Number.isInteger(attempt) || attempt < 1)
|
|
29
|
+
return undefined;
|
|
30
|
+
const relative = parts.slice(5).join("/");
|
|
31
|
+
if (relative.length === 0)
|
|
32
|
+
return undefined;
|
|
33
|
+
return {
|
|
34
|
+
stageId: parts[1],
|
|
35
|
+
attempt,
|
|
36
|
+
relative,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Accept a listed artifact path under a mode.
|
|
41
|
+
* - `grantable`: full run-relative path to a prior stage/attempt artifact file.
|
|
42
|
+
* - `current_attempt`: that attempt's run-relative prefix, or a bare path under
|
|
43
|
+
* its artifacts dir; foreign `stages/...` paths are rejected.
|
|
44
|
+
*/
|
|
45
|
+
export function acceptArtifactListedPath(listedPath, mode) {
|
|
46
|
+
if (mode.kind === "grantable") {
|
|
47
|
+
return parseStrictGrantable(listedPath);
|
|
48
|
+
}
|
|
49
|
+
if (typeof listedPath !== "string" || listedPath.length === 0) {
|
|
50
|
+
return undefined;
|
|
51
|
+
}
|
|
52
|
+
const posixListed = listedPath.replace(/\\/g, "/");
|
|
53
|
+
if (path.isAbsolute(listedPath) || path.posix.isAbsolute(posixListed)) {
|
|
54
|
+
return undefined;
|
|
55
|
+
}
|
|
56
|
+
const prefix = attemptArtifactsPrefix(mode.stageId, mode.attempt);
|
|
57
|
+
if (posixListed === prefix || posixListed === `${prefix}/`) {
|
|
58
|
+
return undefined;
|
|
59
|
+
}
|
|
60
|
+
if (posixListed.startsWith(`${prefix}/`)) {
|
|
61
|
+
const relative = posixListed.slice(prefix.length + 1);
|
|
62
|
+
if (relative.length === 0)
|
|
63
|
+
return undefined;
|
|
64
|
+
return {
|
|
65
|
+
stageId: mode.stageId,
|
|
66
|
+
attempt: mode.attempt,
|
|
67
|
+
relative,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
if (posixListed.startsWith("stages/")) {
|
|
71
|
+
return undefined;
|
|
72
|
+
}
|
|
73
|
+
return {
|
|
74
|
+
stageId: mode.stageId,
|
|
75
|
+
attempt: mode.attempt,
|
|
76
|
+
relative: posixListed,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
export function parseArtifactListedPath(listedPath) {
|
|
80
|
+
return parseStrictGrantable(listedPath);
|
|
81
|
+
}
|
|
82
|
+
export function isGrantableArtifactPath(listedPath) {
|
|
83
|
+
return acceptArtifactListedPath(listedPath, { kind: "grantable" }) !== undefined;
|
|
84
|
+
}
|