stageflow 0.42.0 → 0.46.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -3
- package/dist/agent/activity.d.ts +18 -0
- package/dist/agent/claudeAdapter.js +7 -0
- package/dist/agent/claudeTools.d.ts +27 -18
- package/dist/agent/claudeTools.js +54 -15
- package/dist/agent/compactionEvents.d.ts +34 -0
- package/dist/agent/compactionEvents.js +37 -0
- package/dist/agent/openRouterRouting.d.ts +7 -0
- package/dist/agent/openRouterRouting.js +23 -0
- package/dist/agent/piAdapter.d.ts +6 -0
- package/dist/agent/piAdapter.js +128 -27
- package/dist/agent/piSettings.d.ts +7 -0
- package/dist/agent/piSettings.js +9 -0
- package/dist/agent/sandboxPiTools.d.ts +3 -1
- package/dist/agent/sandboxPiTools.js +8 -2
- package/dist/agent/stageTurnDriver.d.ts +11 -2
- package/dist/agent/stageTurnDriver.js +45 -4
- package/dist/agent/toolOutputBudget.d.ts +15 -1
- package/dist/agent/toolOutputBudget.js +18 -0
- package/dist/agent/toolOutputBudgetExtension.d.ts +2 -0
- package/dist/agent/toolOutputBudgetExtension.js +40 -3
- package/dist/agents/authorChat.d.ts +1 -0
- package/dist/agents/authorChat.js +7 -0
- package/dist/agents/authorSessionStore.d.ts +3 -0
- package/dist/agents/authorSessionStore.js +17 -2
- package/dist/agents/evalGate.js +1 -0
- package/dist/agents/hostContext.d.ts +1 -1
- package/dist/agents/runtime/piRuntime.d.ts +5 -1
- package/dist/agents/runtime/piRuntime.js +46 -12
- package/dist/agents/service.js +3 -2
- package/dist/agents/sessions.js +3 -1
- package/dist/cli/evalCommand.d.ts +1 -1
- package/dist/cli/evalCommand.js +39 -2
- package/dist/cli/hostClient.d.ts +6 -0
- package/dist/cli/hostClient.js +4 -0
- package/dist/compaction/directives.d.ts +18 -0
- package/dist/compaction/directives.js +119 -0
- package/dist/compaction/extension.d.ts +20 -0
- package/dist/compaction/extension.js +78 -0
- package/dist/compaction/mask.d.ts +18 -0
- package/dist/compaction/mask.js +155 -0
- package/dist/compaction/pin.d.ts +21 -0
- package/dist/compaction/pin.js +270 -0
- package/dist/compaction/scrub.d.ts +7 -0
- package/dist/compaction/scrub.js +26 -0
- package/dist/compaction/sessionWiring.d.ts +22 -0
- package/dist/compaction/sessionWiring.js +26 -0
- package/dist/compaction/summarize.d.ts +79 -0
- package/dist/compaction/summarize.js +171 -0
- package/dist/compaction/toolPolicy.d.ts +3 -0
- package/dist/compaction/toolPolicy.js +33 -0
- package/dist/compaction/types.d.ts +45 -0
- package/dist/compaction/types.js +7 -0
- package/dist/compaction/workingState.d.ts +56 -0
- package/dist/compaction/workingState.js +198 -0
- package/dist/config/createPipeline.js +2 -0
- package/dist/config/createTrigger.js +7 -1
- package/dist/config/ensureCatalogScanRoot.d.ts +2 -0
- package/dist/config/ensureCatalogScanRoot.js +6 -0
- package/dist/config/evalCatalog.js +30 -2
- package/dist/config/githubTriggerAction.d.ts +8 -0
- package/dist/config/githubTriggerAction.js +29 -0
- package/dist/config/hostConfig.js +2 -0
- package/dist/config/loadEval.d.ts +3 -1
- package/dist/config/loadEval.js +96 -10
- package/dist/config/pipelineNeeds.d.ts +2 -0
- package/dist/config/pipelineNeeds.js +16 -3
- package/dist/config/pipelineRoute.js +5 -4
- package/dist/config/validateCatalog.d.ts +1 -1
- package/dist/config/validateCatalog.js +12 -0
- package/dist/config/validateEval.js +60 -3
- package/dist/config/writeEval.js +2 -2
- package/dist/envelope/assertEmittedArtifacts.d.ts +12 -0
- package/dist/envelope/assertEmittedArtifacts.js +43 -0
- package/dist/envelope/preEmitChecks.d.ts +3 -0
- package/dist/evals/calibration.d.ts +21 -0
- package/dist/evals/calibration.js +69 -0
- package/dist/evals/calibrationSuggest.d.ts +29 -0
- package/dist/evals/calibrationSuggest.js +72 -0
- package/dist/evals/calibrator.d.ts +46 -0
- package/dist/evals/calibrator.js +148 -0
- package/dist/evals/compare.d.ts +1 -1
- package/dist/evals/compare.js +3 -0
- package/dist/evals/constants.d.ts +7 -3
- package/dist/evals/constants.js +7 -1
- package/dist/evals/engine/FakeEngine.d.ts +2 -1
- package/dist/evals/engine/FakeEngine.js +9 -3
- package/dist/evals/engine/promptfoo/PromptfooEngine.d.ts +1 -1
- package/dist/evals/engine/promptfoo/PromptfooEngine.js +4 -2
- package/dist/evals/engine/promptfoo/generate.d.ts +3 -1
- package/dist/evals/engine/promptfoo/generate.js +43 -23
- package/dist/evals/engine/promptfoo/judgeProviders.js +2 -1
- package/dist/evals/engine/promptfoo/normalize.d.ts +4 -1
- package/dist/evals/engine/promptfoo/normalize.js +16 -3
- package/dist/evals/engine/promptfoo/rubricPrompt.d.ts +3 -1
- package/dist/evals/engine/promptfoo/rubricPrompt.js +15 -5
- package/dist/evals/errors.d.ts +7 -0
- package/dist/evals/errors.js +10 -0
- package/dist/evals/judgeProviders.d.ts +6 -0
- package/dist/evals/judgeProviders.js +14 -0
- package/dist/evals/judged.d.ts +7 -0
- package/dist/evals/judged.js +7 -0
- package/dist/evals/manager.d.ts +27 -8
- package/dist/evals/manager.js +73 -12
- package/dist/evals/plan.d.ts +10 -1
- package/dist/evals/plan.js +33 -13
- package/dist/evals/planInternalAgent.d.ts +19 -0
- package/dist/evals/planInternalAgent.js +201 -0
- package/dist/evals/read/api.d.ts +18 -0
- package/dist/evals/read/api.js +10 -1
- package/dist/evals/read/labelQueue.d.ts +45 -0
- package/dist/evals/read/labelQueue.js +83 -0
- package/dist/evals/read/list.js +10 -1
- package/dist/evals/read/runs.d.ts +5 -0
- package/dist/evals/read/runs.js +7 -1
- package/dist/evals/runner/caseRunner.d.ts +2 -1
- package/dist/evals/runner/caseRunner.js +16 -5
- package/dist/evals/runner/internalAgentCaseRunner.d.ts +6 -0
- package/dist/evals/runner/internalAgentCaseRunner.js +250 -0
- package/dist/evals/snapshot.d.ts +10 -2
- package/dist/evals/snapshot.js +8 -3
- package/dist/evals/stats.d.ts +40 -0
- package/dist/evals/stats.js +69 -0
- package/dist/evals/storeTypes.d.ts +42 -0
- package/dist/evals/types.d.ts +25 -2
- package/dist/evals/types.js +4 -1
- package/dist/evals/votes.d.ts +10 -0
- package/dist/evals/votes.js +60 -0
- package/dist/guide/cardContract.d.ts +133 -0
- package/dist/guide/cardContract.js +156 -0
- package/dist/guide/chat.d.ts +45 -0
- package/dist/guide/chat.js +378 -0
- package/dist/guide/knowledge/index.d.ts +14 -0
- package/dist/guide/knowledge/index.js +109 -0
- package/dist/guide/knowledge/topics/agents.d.ts +2 -0
- package/dist/guide/knowledge/topics/agents.js +58 -0
- package/dist/guide/knowledge/topics/bestPractices.d.ts +2 -0
- package/dist/guide/knowledge/topics/bestPractices.js +67 -0
- package/dist/guide/knowledge/topics/connections.d.ts +2 -0
- package/dist/guide/knowledge/topics/connections.js +66 -0
- package/dist/guide/knowledge/topics/consoleMap.d.ts +2 -0
- package/dist/guide/knowledge/topics/consoleMap.js +55 -0
- package/dist/guide/knowledge/topics/evals.d.ts +2 -0
- package/dist/guide/knowledge/topics/evals.js +72 -0
- package/dist/guide/knowledge/topics/limitations.d.ts +2 -0
- package/dist/guide/knowledge/topics/limitations.js +84 -0
- package/dist/guide/knowledge/topics/mcpSkillsBrowser.d.ts +2 -0
- package/dist/guide/knowledge/topics/mcpSkillsBrowser.js +57 -0
- package/dist/guide/knowledge/topics/memories.d.ts +2 -0
- package/dist/guide/knowledge/topics/memories.js +63 -0
- package/dist/guide/knowledge/topics/modelsAndProviders.d.ts +2 -0
- package/dist/guide/knowledge/topics/modelsAndProviders.js +56 -0
- package/dist/guide/knowledge/topics/overview.d.ts +2 -0
- package/dist/guide/knowledge/topics/overview.js +56 -0
- package/dist/guide/knowledge/topics/pipelines.d.ts +2 -0
- package/dist/guide/knowledge/topics/pipelines.js +93 -0
- package/dist/guide/knowledge/topics/stages.d.ts +2 -0
- package/dist/guide/knowledge/topics/stages.js +54 -0
- package/dist/guide/knowledge/topics/tasks.d.ts +2 -0
- package/dist/guide/knowledge/topics/tasks.js +52 -0
- package/dist/guide/knowledge/topics/triggers.d.ts +2 -0
- package/dist/guide/knowledge/topics/triggers.js +68 -0
- package/dist/guide/knowledge/topics/troubleshooting.d.ts +2 -0
- package/dist/guide/knowledge/topics/troubleshooting.js +85 -0
- package/dist/guide/knowledgeTools.d.ts +2 -0
- package/dist/guide/knowledgeTools.js +56 -0
- package/dist/guide/showUi.d.ts +6 -0
- package/dist/guide/showUi.js +546 -0
- package/dist/guide/threadStore.d.ts +48 -0
- package/dist/guide/threadStore.js +189 -0
- package/dist/guide/writeRoot.d.ts +11 -0
- package/dist/guide/writeRoot.js +70 -0
- package/dist/hosted/translate.js +2 -0
- package/dist/mcp/agentTools.js +12 -0
- package/dist/mcp/catalogTools.js +18 -20
- package/dist/mcp/catalogWriteTools.d.ts +8 -0
- package/dist/mcp/catalogWriteTools.js +225 -0
- package/dist/mcp/connectionTools.d.ts +3 -0
- package/dist/mcp/connectionTools.js +85 -0
- package/dist/mcp/controlTools.js +34 -1
- package/dist/mcp/deps.d.ts +9 -0
- package/dist/mcp/evalTools.js +723 -21
- package/dist/mcp/memoryHostTools.d.ts +3 -0
- package/dist/mcp/memoryHostTools.js +72 -0
- package/dist/mcp/pipelineAuthorTools.js +14 -1
- package/dist/mcp/server.js +5 -0
- package/dist/mcp/tools.js +6 -0
- package/dist/mcp/triggerTools.js +77 -31
- package/dist/operatorAgent/evalGate.d.ts +22 -0
- package/dist/operatorAgent/evalGate.js +77 -0
- package/dist/operatorAgent/fakeHost.js +10 -2
- package/dist/operatorAgent/host.js +18 -2
- package/dist/operatorAgent/index.d.ts +1 -0
- package/dist/operatorAgent/index.js +1 -0
- package/dist/operatorAgent/internalAgents.d.ts +11 -0
- package/dist/operatorAgent/internalAgents.js +13 -0
- package/dist/operatorAgent/piModel.d.ts +5 -1
- package/dist/operatorAgent/piModel.js +15 -10
- package/dist/operatorAgent/profiles/agentAuthor.js +66 -22
- package/dist/operatorAgent/profiles/agentAuthorPlaybook.d.ts +1 -1
- package/dist/operatorAgent/profiles/agentAuthorPlaybook.js +1 -1
- package/dist/operatorAgent/profiles/guideConfirm.d.ts +21 -0
- package/dist/operatorAgent/profiles/guideConfirm.js +100 -0
- package/dist/operatorAgent/profiles/guideTools.d.ts +7 -0
- package/dist/operatorAgent/profiles/guideTools.js +123 -0
- package/dist/operatorAgent/profiles/stageflowGuide.d.ts +7 -0
- package/dist/operatorAgent/profiles/stageflowGuide.js +133 -0
- package/dist/operatorAgent/profiles/workshopAuthor.js +23 -0
- package/dist/operatorAgent/profiles/workshopAuthorPlaybook.d.ts +1 -1
- package/dist/operatorAgent/profiles/workshopAuthorPlaybook.js +9 -1
- package/dist/operatorAgent/profiles/workshopAuthorTools.d.ts +2 -0
- package/dist/operatorAgent/profiles/workshopAuthorTools.js +197 -9
- package/dist/operatorAgent/types.d.ts +23 -0
- package/dist/package-meta.d.ts +1 -1
- package/dist/package-meta.js +1 -1
- package/dist/runstore/port.d.ts +13 -1
- package/dist/runstore/retention.d.ts +2 -1
- package/dist/runstore/retention.js +3 -7
- package/dist/runstore/runStatusClock.d.ts +17 -0
- package/dist/runstore/runStatusClock.js +19 -0
- package/dist/runstore/sqlite/SqliteRunStore.d.ts +11 -1
- package/dist/runstore/sqlite/SqliteRunStore.js +70 -45
- package/dist/runstore/sqlite/calibrationStore.d.ts +10 -0
- package/dist/runstore/sqlite/calibrationStore.js +106 -0
- package/dist/runstore/sqlite/evalStore.d.ts +5 -0
- package/dist/runstore/sqlite/evalStore.js +7 -0
- package/dist/runstore/sqlite/migrations/018-eval-grader-labels.d.ts +11 -0
- package/dist/runstore/sqlite/migrations/018-eval-grader-labels.js +43 -0
- package/dist/runstore/sqlite/migrations/index.js +2 -0
- package/dist/runstore/workspaceLayout.js +6 -1
- package/dist/runtime/artifactDiskInspect.d.ts +14 -0
- package/dist/runtime/artifactDiskInspect.js +52 -0
- package/dist/runtime/artifactPath.d.ts +24 -0
- package/dist/runtime/artifactPath.js +84 -0
- package/dist/runtime/completionCheckRunner.js +15 -43
- package/dist/runtime/githubPollSource.d.ts +22 -4
- package/dist/runtime/githubPollSource.js +93 -8
- package/dist/runtime/priorArtifactGrants.d.ts +8 -0
- package/dist/runtime/priorArtifactGrants.js +105 -0
- package/dist/runtime/runActionEligibility.d.ts +46 -0
- package/dist/runtime/runActionEligibility.js +123 -0
- package/dist/runtime/runManager.d.ts +4 -1
- package/dist/runtime/runManager.js +52 -10
- package/dist/runtime/runRetryCoordinator.d.ts +2 -14
- package/dist/runtime/runRetryCoordinator.js +3 -67
- package/dist/runtime/sandboxContainer.d.ts +7 -0
- package/dist/runtime/sandboxContainer.js +48 -6
- package/dist/runtime/sandboxDesk.d.ts +2 -1
- package/dist/runtime/sandboxDesk.js +11 -12
- package/dist/runtime/stageEnvironment.js +1 -0
- package/dist/runtime/stageRoots.d.ts +12 -0
- package/dist/runtime/stageSandbox.d.ts +2 -1
- package/dist/runtime/stageSandbox.js +18 -2
- package/dist/runtime/stageWorker.js +20 -0
- package/dist/server/agentAuthorRoutes.js +12 -3
- package/dist/server/bootstrap.d.ts +5 -0
- package/dist/server/bootstrap.js +14 -1
- package/dist/server/evalRoutes.js +251 -12
- package/dist/server/guideRoutes.d.ts +5 -0
- package/dist/server/guideRoutes.js +166 -0
- package/dist/server/http.d.ts +10 -0
- package/dist/server/http.js +35 -1
- package/dist/server/internalAgentRoutes.d.ts +6 -0
- package/dist/server/internalAgentRoutes.js +12 -0
- package/dist/server/mcpHost.js +2 -0
- package/dist/toolkit/meta.d.ts +1 -1
- package/dist/toolkit/meta.js +41 -1
- package/dist/toolkit/profiles.d.ts +5 -0
- package/dist/toolkit/profiles.js +12 -0
- package/dist/tools/askOperator.js +2 -1
- package/dist/tools/emitStageEnvelope.d.ts +17 -16
- package/dist/tools/emitStageEnvelope.js +46 -32
- package/dist/tools/stageToolNames.d.ts +4 -0
- package/dist/tools/stageToolNames.js +4 -0
- package/dist/tools/workingState.d.ts +40 -0
- package/dist/tools/workingState.js +108 -0
- package/dist/tools/writeStageArtifact.d.ts +0 -9
- package/dist/tools/writeStageArtifact.js +2 -1
- package/dist/ui/assets/{MarkdownText-BTrOqu5J.js → MarkdownText-BS2BBbCx.js} +2 -2
- package/dist/ui/assets/adapters-DPD-cAsy.js +17 -0
- package/dist/ui/assets/chat-BD1-VTT6.js +2 -0
- package/dist/ui/assets/chat-DXnQv9st.css +2 -0
- package/dist/ui/assets/main-AX5XNLtX.js +278 -0
- package/dist/ui/assets/{main-CHlmTNJe.css → main-CFX0_n8J.css} +1 -1
- package/dist/ui/chat.html +3 -4
- package/dist/ui/index.html +3 -4
- package/dist/workshop/chatTurn.js +1 -1
- package/dist/workshop/workshopPin.d.ts +3 -3
- package/dist/workshop/workshopPin.js +16 -15
- package/examples/compaction-stress/README.md +30 -0
- package/examples/compaction-stress/check-run.mjs +53 -0
- package/examples/compaction-stress/collect-evidence.yaml +60 -0
- package/examples/compaction-stress/compaction-stress.pipeline.yaml +141 -0
- package/examples/compaction-stress/compaction-stress.task.yaml +15 -0
- package/examples/compaction-stress/generate-evidence.mjs +110 -0
- package/examples/compaction-stress/root-cause.yaml +26 -0
- package/examples/compaction-stress/stageflow.yaml +9 -0
- package/examples/compaction-stress/write-postmortem.yaml +40 -0
- package/examples/evals/README.md +18 -0
- package/examples/evals/guide-behavior.dataset.jsonl +15 -0
- package/examples/evals/guide-behavior.eval.yaml +35 -0
- package/examples/evals/guide-depth.dataset.jsonl +35 -0
- package/examples/evals/guide-depth.eval.yaml +169 -0
- package/examples/memory/evals/guide-behavior.dataset.jsonl +15 -0
- package/examples/memory/evals/guide-behavior.eval.yaml +35 -0
- package/examples/memory/evals/guide-depth.dataset.jsonl +35 -0
- package/examples/memory/evals/guide-depth.eval.yaml +169 -0
- package/package.json +1 -1
- package/skills/stageflow/references/control-surface.md +1 -1
- package/skills/stageflow/references/evals.md +1 -0
- package/dist/ui/assets/AssistantRuntimeProvider-qIX1xNoH.js +0 -9
- package/dist/ui/assets/ReadonlyStore-BfXqHykm.js +0 -6
- package/dist/ui/assets/chat-Df_wCpak.js +0 -5
- package/dist/ui/assets/chat-P7KQznHm.css +0 -2
- package/dist/ui/assets/main-CvVfxmMd.js +0 -271
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { isRunClockTerminal, } from "./runStatusClock.js";
|
|
1
2
|
export const DAY_MS = 24 * 60 * 60 * 1000;
|
|
2
3
|
/** Defaults: succeeded 3d/30d; failed and cancelled 30d/90d (R30 / AE6). */
|
|
3
4
|
export const DEFAULT_RETENTION_WINDOWS = {
|
|
@@ -7,11 +8,6 @@ export const DEFAULT_RETENTION_WINDOWS = {
|
|
|
7
8
|
};
|
|
8
9
|
export const DEFAULT_BARE_CACHE_TTL_MS = 30 * DAY_MS;
|
|
9
10
|
export const DEFAULT_SLIM_ARTIFACT_MAX_BYTES = 1024 * 1024;
|
|
10
|
-
const TERMINAL = new Set([
|
|
11
|
-
"succeeded",
|
|
12
|
-
"failed",
|
|
13
|
-
"cancelled",
|
|
14
|
-
]);
|
|
15
11
|
function parsePositiveMs(raw) {
|
|
16
12
|
if (raw === undefined)
|
|
17
13
|
return undefined;
|
|
@@ -52,7 +48,7 @@ export function slimArtifactMaxBytesFromEnv(env = process.env) {
|
|
|
52
48
|
*/
|
|
53
49
|
export function retentionDecision(row, now, windows = DEFAULT_RETENTION_WINDOWS) {
|
|
54
50
|
const status = row.status;
|
|
55
|
-
if (
|
|
51
|
+
if (!isRunClockTerminal(status))
|
|
56
52
|
return "none";
|
|
57
53
|
if (row.finished_at === undefined || row.finished_at === "")
|
|
58
54
|
return "none";
|
|
@@ -70,5 +66,5 @@ export function retentionDecision(row, now, windows = DEFAULT_RETENTION_WINDOWS)
|
|
|
70
66
|
return "none";
|
|
71
67
|
}
|
|
72
68
|
export function isTerminalRunStatus(status) {
|
|
73
|
-
return
|
|
69
|
+
return isRunClockTerminal(status);
|
|
74
70
|
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export declare const RUN_CLOCK_TERMINAL_STATUSES: readonly ["succeeded", "failed", "cancelled"];
|
|
2
|
+
export type RunClockTerminalStatus = (typeof RUN_CLOCK_TERMINAL_STATUSES)[number];
|
|
3
|
+
export type RunClockStatus = "created" | "queued" | "running" | RunClockTerminalStatus;
|
|
4
|
+
export declare function isRunClockTerminal(status: string | undefined): status is RunClockTerminalStatus;
|
|
5
|
+
export type FinishedAtDecision = {
|
|
6
|
+
action: "stamp";
|
|
7
|
+
finishedAt: string;
|
|
8
|
+
} | {
|
|
9
|
+
action: "keep";
|
|
10
|
+
} | {
|
|
11
|
+
action: "clear";
|
|
12
|
+
};
|
|
13
|
+
export declare function nextFinishedAt(input: {
|
|
14
|
+
nextStatus: RunClockStatus;
|
|
15
|
+
currentFinishedAt: string | null | undefined;
|
|
16
|
+
now: string;
|
|
17
|
+
}): FinishedAtDecision;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export const RUN_CLOCK_TERMINAL_STATUSES = [
|
|
2
|
+
"succeeded",
|
|
3
|
+
"failed",
|
|
4
|
+
"cancelled",
|
|
5
|
+
];
|
|
6
|
+
const TERMINAL = new Set(RUN_CLOCK_TERMINAL_STATUSES);
|
|
7
|
+
export function isRunClockTerminal(status) {
|
|
8
|
+
return status !== undefined && TERMINAL.has(status);
|
|
9
|
+
}
|
|
10
|
+
export function nextFinishedAt(input) {
|
|
11
|
+
if (isRunClockTerminal(input.nextStatus)) {
|
|
12
|
+
if (input.currentFinishedAt != null &&
|
|
13
|
+
input.currentFinishedAt !== "") {
|
|
14
|
+
return { action: "keep" };
|
|
15
|
+
}
|
|
16
|
+
return { action: "stamp", finishedAt: input.now };
|
|
17
|
+
}
|
|
18
|
+
return { action: "clear" };
|
|
19
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import Database from "better-sqlite3";
|
|
2
|
-
import type { CreateEvalRunInput, EvalCaseInput, EvalCaseRow, EvalGraderVersionInput, EvalResultInput, EvalResultWithGrades, EvalReviewPatch, EvalReviewRow, EvalRunPage, EvalRunProgress, EvalRunRow, FinishEvalRunInput, ListEvalRunsQuery } from "../../evals/storeTypes.js";
|
|
2
|
+
import type { CreateEvalRunInput, EvalCaseInput, EvalCaseRow, EvalGraderVersionInput, EvalResultInput, EvalResultWithGrades, EvalReviewPatch, EvalCalibrationInput, EvalCalibrationRow, EvalGraderLabelPatch, EvalGraderLabelRow, EvalReviewRow, EvalRunPage, EvalRunProgress, EvalRunRow, FinishEvalRunInput, ListEvalRunsQuery } from "../../evals/storeTypes.js";
|
|
3
3
|
import type { EvalTargetKind } from "../../evals/constants.js";
|
|
4
4
|
import type { StageEnvelope } from "../../types/envelope.js";
|
|
5
5
|
import type { StageLogLine } from "../../agent/activity.js";
|
|
@@ -32,6 +32,7 @@ export declare class SqliteRunStore implements RunStore {
|
|
|
32
32
|
findRunByDispatchKey(dispatchKey: string): Promise<CreatedRun | undefined>;
|
|
33
33
|
private readSubmission;
|
|
34
34
|
getRunBySubmission(key: string): Promise<RunSubmissionRecord | null>;
|
|
35
|
+
private applyRunStatusUpdate;
|
|
35
36
|
updateRunStatus(runId: string, status: RunStatus): Promise<void>;
|
|
36
37
|
tryUpdateRunStatus(runId: string, status: RunStatus, expectedStatus: RunStatus): Promise<boolean>;
|
|
37
38
|
patchRunWorkspaceBinding(runId: string, patch: {
|
|
@@ -95,6 +96,15 @@ export declare class SqliteRunStore implements RunStore {
|
|
|
95
96
|
getEvalCases(evalRunId: string): Promise<EvalCaseRow[]>;
|
|
96
97
|
putEvalReview(evalRunId: string, caseId: string, repeatIndex: number, patch: EvalReviewPatch): Promise<EvalReviewRow>;
|
|
97
98
|
getEvalReviews(evalRunId: string): Promise<EvalReviewRow[]>;
|
|
99
|
+
putEvalGraderLabel(evalRunId: string, caseId: string, repeatIndex: number, graderId: string, patch: EvalGraderLabelPatch): Promise<EvalGraderLabelRow | null>;
|
|
100
|
+
listGradedResultKeys(evalRunId: string, graderId: string): Promise<Array<{
|
|
101
|
+
caseId: string;
|
|
102
|
+
repeatIndex: number;
|
|
103
|
+
}>>;
|
|
104
|
+
listEvalGraderLabels(evalRunId: string): Promise<EvalGraderLabelRow[]>;
|
|
105
|
+
listEvalGraderLabelsForEval(projectRoot: string, evalId: string, graderId: string): Promise<EvalGraderLabelRow[]>;
|
|
106
|
+
insertEvalCalibration(input: EvalCalibrationInput): Promise<EvalCalibrationRow>;
|
|
107
|
+
listEvalCalibrations(projectRoot: string, evalId: string, graderId?: string, includeCandidates?: boolean, limit?: number): Promise<EvalCalibrationRow[]>;
|
|
98
108
|
listFailureTypes(projectRoot: string, evalId: string): Promise<string[]>;
|
|
99
109
|
recordGraderVersions(projectRoot: string, evalId: string, graders: EvalGraderVersionInput[]): Promise<Record<string, number>>;
|
|
100
110
|
listEvalSubjectRefs(evalRunId: string, targetKind: EvalTargetKind): Promise<string[]>;
|
|
@@ -10,11 +10,13 @@ import { newRunId, runWorkspaceDir } from "../paths.js";
|
|
|
10
10
|
import { attemptAgentDir, attemptArtifactsDir, stageDir, } from "../workspaceLayout.js";
|
|
11
11
|
import { importDiskRunsIfEmpty } from "./migrateFromDisk.js";
|
|
12
12
|
import * as evalStore from "./evalStore.js";
|
|
13
|
+
import * as calibrationStore from "./calibrationStore.js";
|
|
13
14
|
import * as suggestionStore from "./suggestionStore.js";
|
|
14
15
|
import { applyPendingMigrations, assertSchemaVersion, } from "./migrations/index.js";
|
|
15
16
|
import { StoreSchemaError } from "./storeSchemaError.js";
|
|
16
17
|
import { applyStorePragmas } from "./applyStorePragmas.js";
|
|
17
18
|
import { RunSubmissionExistsError } from "../submission.js";
|
|
19
|
+
import { nextFinishedAt } from "../runStatusClock.js";
|
|
18
20
|
const EXECUTION_SELECT_COLS = `run_id, stage_id, attempt, status, verification_outcome, started_at, finished_at, envelope_json, cost_usd, usage_json, auto_resume_count`;
|
|
19
21
|
const ensuredStageDirs = new Set();
|
|
20
22
|
function executionFromRow(row) {
|
|
@@ -289,58 +291,57 @@ export class SqliteRunStore {
|
|
|
289
291
|
await this.ready();
|
|
290
292
|
return this.readSubmission(key);
|
|
291
293
|
}
|
|
294
|
+
applyRunStatusUpdate(runId, status, expectedStatus) {
|
|
295
|
+
const now = new Date().toISOString();
|
|
296
|
+
const current = this.db
|
|
297
|
+
.prepare(`SELECT finished_at FROM runs WHERE run_id = ?`)
|
|
298
|
+
.get(runId);
|
|
299
|
+
if (current === undefined) {
|
|
300
|
+
throw new Error(`Run not found: ${runId}`);
|
|
301
|
+
}
|
|
302
|
+
const decision = nextFinishedAt({
|
|
303
|
+
nextStatus: status,
|
|
304
|
+
currentFinishedAt: current.finished_at,
|
|
305
|
+
now,
|
|
306
|
+
});
|
|
307
|
+
const params = {
|
|
308
|
+
run_id: runId,
|
|
309
|
+
status,
|
|
310
|
+
updated_at: now,
|
|
311
|
+
};
|
|
312
|
+
let finishedSql;
|
|
313
|
+
if (decision.action === "stamp") {
|
|
314
|
+
finishedSql = "finished_at = @finished_at";
|
|
315
|
+
params.finished_at = decision.finishedAt;
|
|
316
|
+
}
|
|
317
|
+
else if (decision.action === "clear") {
|
|
318
|
+
finishedSql = "finished_at = NULL";
|
|
319
|
+
}
|
|
320
|
+
else {
|
|
321
|
+
finishedSql = "finished_at = finished_at";
|
|
322
|
+
}
|
|
323
|
+
const expectedClause = expectedStatus !== undefined
|
|
324
|
+
? " AND status = @expected_status"
|
|
325
|
+
: "";
|
|
326
|
+
if (expectedStatus !== undefined) {
|
|
327
|
+
params.expected_status = expectedStatus;
|
|
328
|
+
}
|
|
329
|
+
const result = this.db
|
|
330
|
+
.prepare(`UPDATE runs SET status = @status, updated_at = @updated_at, ${finishedSql}
|
|
331
|
+
WHERE run_id = @run_id${expectedClause}`)
|
|
332
|
+
.run(params);
|
|
333
|
+
return result.changes;
|
|
334
|
+
}
|
|
292
335
|
async updateRunStatus(runId, status) {
|
|
293
336
|
await this.ready();
|
|
294
|
-
|
|
295
|
-
const isTerminal = status === "succeeded" || status === "failed" || status === "cancelled";
|
|
296
|
-
const result = isTerminal
|
|
297
|
-
? this.db
|
|
298
|
-
.prepare(`UPDATE runs SET status = @status, updated_at = @updated_at,
|
|
299
|
-
finished_at = COALESCE(finished_at, @finished_at)
|
|
300
|
-
WHERE run_id = @run_id`)
|
|
301
|
-
.run({
|
|
302
|
-
run_id: runId,
|
|
303
|
-
status,
|
|
304
|
-
updated_at: now,
|
|
305
|
-
finished_at: now,
|
|
306
|
-
})
|
|
307
|
-
: this.db
|
|
308
|
-
.prepare(`UPDATE runs SET status = @status, updated_at = @updated_at WHERE run_id = @run_id`)
|
|
309
|
-
.run({
|
|
310
|
-
run_id: runId,
|
|
311
|
-
status,
|
|
312
|
-
updated_at: now,
|
|
313
|
-
});
|
|
314
|
-
if (result.changes === 0) {
|
|
337
|
+
if (this.applyRunStatusUpdate(runId, status, undefined) === 0) {
|
|
315
338
|
throw new Error(`Run not found: ${runId}`);
|
|
316
339
|
}
|
|
317
340
|
}
|
|
318
341
|
async tryUpdateRunStatus(runId, status, expectedStatus) {
|
|
319
342
|
await this.ready();
|
|
320
|
-
const
|
|
321
|
-
|
|
322
|
-
const result = isTerminal
|
|
323
|
-
? this.db
|
|
324
|
-
.prepare(`UPDATE runs SET status = @status, updated_at = @updated_at,
|
|
325
|
-
finished_at = COALESCE(finished_at, @finished_at)
|
|
326
|
-
WHERE run_id = @run_id AND status = @expected_status`)
|
|
327
|
-
.run({
|
|
328
|
-
run_id: runId,
|
|
329
|
-
status,
|
|
330
|
-
expected_status: expectedStatus,
|
|
331
|
-
updated_at: now,
|
|
332
|
-
finished_at: now,
|
|
333
|
-
})
|
|
334
|
-
: this.db
|
|
335
|
-
.prepare(`UPDATE runs SET status = @status, updated_at = @updated_at
|
|
336
|
-
WHERE run_id = @run_id AND status = @expected_status`)
|
|
337
|
-
.run({
|
|
338
|
-
run_id: runId,
|
|
339
|
-
status,
|
|
340
|
-
expected_status: expectedStatus,
|
|
341
|
-
updated_at: now,
|
|
342
|
-
});
|
|
343
|
-
if (result.changes > 0)
|
|
343
|
+
const changes = this.applyRunStatusUpdate(runId, status, expectedStatus);
|
|
344
|
+
if (changes > 0)
|
|
344
345
|
return true;
|
|
345
346
|
const exists = this.db
|
|
346
347
|
.prepare(`SELECT 1 AS ok FROM runs WHERE run_id = ?`)
|
|
@@ -949,6 +950,30 @@ export class SqliteRunStore {
|
|
|
949
950
|
await this.ready();
|
|
950
951
|
return evalStore.getEvalReviews(this.db, evalRunId);
|
|
951
952
|
}
|
|
953
|
+
async putEvalGraderLabel(evalRunId, caseId, repeatIndex, graderId, patch) {
|
|
954
|
+
await this.ready();
|
|
955
|
+
return calibrationStore.putEvalGraderLabel(this.db, evalRunId, caseId, repeatIndex, graderId, patch);
|
|
956
|
+
}
|
|
957
|
+
async listGradedResultKeys(evalRunId, graderId) {
|
|
958
|
+
await this.ready();
|
|
959
|
+
return evalStore.listGradedResultKeys(this.db, evalRunId, graderId);
|
|
960
|
+
}
|
|
961
|
+
async listEvalGraderLabels(evalRunId) {
|
|
962
|
+
await this.ready();
|
|
963
|
+
return calibrationStore.listEvalGraderLabels(this.db, evalRunId);
|
|
964
|
+
}
|
|
965
|
+
async listEvalGraderLabelsForEval(projectRoot, evalId, graderId) {
|
|
966
|
+
await this.ready();
|
|
967
|
+
return calibrationStore.listEvalGraderLabelsForEval(this.db, projectRoot, evalId, graderId);
|
|
968
|
+
}
|
|
969
|
+
async insertEvalCalibration(input) {
|
|
970
|
+
await this.ready();
|
|
971
|
+
return calibrationStore.insertEvalCalibration(this.db, input);
|
|
972
|
+
}
|
|
973
|
+
async listEvalCalibrations(projectRoot, evalId, graderId, includeCandidates, limit) {
|
|
974
|
+
await this.ready();
|
|
975
|
+
return calibrationStore.listEvalCalibrations(this.db, projectRoot, evalId, graderId, includeCandidates, limit);
|
|
976
|
+
}
|
|
952
977
|
async listFailureTypes(projectRoot, evalId) {
|
|
953
978
|
await this.ready();
|
|
954
979
|
return evalStore.listFailureTypes(this.db, projectRoot, evalId);
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type Database from "better-sqlite3";
|
|
2
|
+
import type { EvalCalibrationInput, EvalCalibrationRow, EvalGraderLabelPatch, EvalGraderLabelRow } from "../../evals/storeTypes.js";
|
|
3
|
+
/** Sets, changes or (label null) clears one grader label on a stored result. */
|
|
4
|
+
export declare function putEvalGraderLabel(db: Database.Database, evalRunId: string, caseId: string, repeatIndex: number, graderId: string, patch: EvalGraderLabelPatch): EvalGraderLabelRow | null;
|
|
5
|
+
export declare function listEvalGraderLabels(db: Database.Database, evalRunId: string): EvalGraderLabelRow[];
|
|
6
|
+
/** Labels across every run of one eval, newest label first. */
|
|
7
|
+
export declare function listEvalGraderLabelsForEval(db: Database.Database, projectRoot: string, evalId: string, graderId: string): EvalGraderLabelRow[];
|
|
8
|
+
export declare function insertEvalCalibration(db: Database.Database, input: EvalCalibrationInput): EvalCalibrationRow;
|
|
9
|
+
/** Newest first. Candidate (unsaved-rubric) calibrations are included only on request. */
|
|
10
|
+
export declare function listEvalCalibrations(db: Database.Database, projectRoot: string, evalId: string, graderId?: string, includeCandidates?: boolean, limit?: number): EvalCalibrationRow[];
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { randomBytes } from "node:crypto";
|
|
2
|
+
import { EVAL_LIMITS } from "../../evals/constants.js";
|
|
3
|
+
import { EvalStoreError } from "../../evals/storeError.js";
|
|
4
|
+
const now = () => new Date().toISOString();
|
|
5
|
+
const labelFromDb = (r) => ({
|
|
6
|
+
evalRunId: r.eval_run_id,
|
|
7
|
+
caseId: r.case_id,
|
|
8
|
+
repeatIndex: r.repeat_index,
|
|
9
|
+
graderId: r.grader_id,
|
|
10
|
+
label: r.label,
|
|
11
|
+
note: r.note,
|
|
12
|
+
createdAt: r.created_at,
|
|
13
|
+
updatedAt: r.updated_at,
|
|
14
|
+
});
|
|
15
|
+
/** Sets, changes or (label null) clears one grader label on a stored result. */
|
|
16
|
+
export function putEvalGraderLabel(db, evalRunId, caseId, repeatIndex, graderId, patch) {
|
|
17
|
+
if (typeof patch.note === "string" && patch.note.length > EVAL_LIMITS.reviewNoteMaxChars) {
|
|
18
|
+
throw new Error(`Label note must be at most ${EVAL_LIMITS.reviewNoteMaxChars} characters`);
|
|
19
|
+
}
|
|
20
|
+
const exists = db
|
|
21
|
+
.prepare(`SELECT 1 AS ok FROM eval_results WHERE eval_run_id = ? AND case_id = ? AND repeat_index = ?`)
|
|
22
|
+
.get(evalRunId, caseId, repeatIndex);
|
|
23
|
+
if (!exists)
|
|
24
|
+
throw new EvalStoreError("EVAL_RUN_NOT_FOUND", `No result for ${caseId}#${repeatIndex} in ${evalRunId}`);
|
|
25
|
+
if (patch.label === null) {
|
|
26
|
+
db.prepare(`DELETE FROM eval_grader_labels WHERE eval_run_id = ? AND case_id = ? AND repeat_index = ? AND grader_id = ?`).run(evalRunId, caseId, repeatIndex, graderId);
|
|
27
|
+
return null;
|
|
28
|
+
}
|
|
29
|
+
const at = now();
|
|
30
|
+
db.prepare(`INSERT INTO eval_grader_labels (eval_run_id, case_id, repeat_index, grader_id, label, note, created_at, updated_at)
|
|
31
|
+
VALUES (@run, @case, @repeat, @grader, @label, @note, @at, @at)
|
|
32
|
+
ON CONFLICT (eval_run_id, case_id, repeat_index, grader_id) DO UPDATE SET
|
|
33
|
+
label = excluded.label,
|
|
34
|
+
note = ${patch.note === undefined ? "eval_grader_labels.note" : "excluded.note"},
|
|
35
|
+
updated_at = excluded.updated_at`).run({
|
|
36
|
+
run: evalRunId,
|
|
37
|
+
case: caseId,
|
|
38
|
+
repeat: repeatIndex,
|
|
39
|
+
grader: graderId,
|
|
40
|
+
label: patch.label,
|
|
41
|
+
note: patch.note ?? null,
|
|
42
|
+
at,
|
|
43
|
+
});
|
|
44
|
+
return labelFromDb(db
|
|
45
|
+
.prepare(`SELECT * FROM eval_grader_labels WHERE eval_run_id = ? AND case_id = ? AND repeat_index = ? AND grader_id = ?`)
|
|
46
|
+
.get(evalRunId, caseId, repeatIndex, graderId));
|
|
47
|
+
}
|
|
48
|
+
export function listEvalGraderLabels(db, evalRunId) {
|
|
49
|
+
return db
|
|
50
|
+
.prepare(`SELECT * FROM eval_grader_labels WHERE eval_run_id = ? ORDER BY case_id, repeat_index, grader_id`)
|
|
51
|
+
.all(evalRunId).map(labelFromDb);
|
|
52
|
+
}
|
|
53
|
+
/** Labels across every run of one eval, newest label first. */
|
|
54
|
+
export function listEvalGraderLabelsForEval(db, projectRoot, evalId, graderId) {
|
|
55
|
+
return db
|
|
56
|
+
.prepare(`SELECT l.* FROM eval_grader_labels l
|
|
57
|
+
JOIN eval_runs r ON r.eval_run_id = l.eval_run_id
|
|
58
|
+
WHERE r.project_root = ? AND r.eval_id = ? AND l.grader_id = ?
|
|
59
|
+
ORDER BY l.updated_at DESC, l.eval_run_id, l.case_id, l.repeat_index`)
|
|
60
|
+
.all(projectRoot, evalId, graderId).map(labelFromDb);
|
|
61
|
+
}
|
|
62
|
+
const calibrationFromDb = (r) => ({
|
|
63
|
+
id: r.id,
|
|
64
|
+
projectRoot: r.project_root,
|
|
65
|
+
evalId: r.eval_id,
|
|
66
|
+
graderId: r.grader_id,
|
|
67
|
+
graderHash: r.grader_hash,
|
|
68
|
+
judge: r.judge,
|
|
69
|
+
promptVersion: r.prompt_version,
|
|
70
|
+
split: r.split,
|
|
71
|
+
candidate: r.candidate === 1,
|
|
72
|
+
stats: JSON.parse(r.stats_json),
|
|
73
|
+
disagreements: JSON.parse(r.disagreements_json),
|
|
74
|
+
costUsd: r.cost_usd,
|
|
75
|
+
createdAt: r.created_at,
|
|
76
|
+
});
|
|
77
|
+
export function insertEvalCalibration(db, input) {
|
|
78
|
+
const id = `cal_${randomBytes(6).toString("hex")}`;
|
|
79
|
+
const createdAt = now();
|
|
80
|
+
db.prepare(`INSERT INTO eval_calibrations
|
|
81
|
+
(id, project_root, eval_id, grader_id, grader_hash, judge, prompt_version, split, candidate, stats_json, disagreements_json, cost_usd, created_at)
|
|
82
|
+
VALUES (@id, @project_root, @eval_id, @grader_id, @grader_hash, @judge, @prompt_version, @split, @candidate, @stats_json, @disagreements_json, @cost_usd, @created_at)`).run({
|
|
83
|
+
id,
|
|
84
|
+
project_root: input.projectRoot,
|
|
85
|
+
eval_id: input.evalId,
|
|
86
|
+
grader_id: input.graderId,
|
|
87
|
+
grader_hash: input.graderHash,
|
|
88
|
+
judge: input.judge,
|
|
89
|
+
prompt_version: input.promptVersion,
|
|
90
|
+
split: input.split,
|
|
91
|
+
candidate: input.candidate ? 1 : 0,
|
|
92
|
+
stats_json: JSON.stringify(input.stats),
|
|
93
|
+
disagreements_json: JSON.stringify(input.disagreements),
|
|
94
|
+
cost_usd: input.costUsd,
|
|
95
|
+
created_at: createdAt,
|
|
96
|
+
});
|
|
97
|
+
return { ...input, id, createdAt };
|
|
98
|
+
}
|
|
99
|
+
/** Newest first. Candidate (unsaved-rubric) calibrations are included only on request. */
|
|
100
|
+
export function listEvalCalibrations(db, projectRoot, evalId, graderId, includeCandidates = false, limit = 20) {
|
|
101
|
+
return db
|
|
102
|
+
.prepare(`SELECT * FROM eval_calibrations
|
|
103
|
+
WHERE project_root = ? AND eval_id = ? AND (? IS NULL OR grader_id = ?) AND (candidate = 0 OR ?)
|
|
104
|
+
ORDER BY created_at DESC LIMIT ?`)
|
|
105
|
+
.all(projectRoot, evalId, graderId ?? null, graderId ?? null, includeCandidates ? 1 : 0, limit).map(calibrationFromDb);
|
|
106
|
+
}
|
|
@@ -10,6 +10,11 @@ export declare function getEvalRun(db: Database.Database, evalRunId: string): Ev
|
|
|
10
10
|
export declare function getEvalRunByNumber(db: Database.Database, projectRoot: string, evalId: string, runNumber: number): EvalRunRow | null;
|
|
11
11
|
export declare function listEvalRuns(db: Database.Database, query: ListEvalRunsQuery): EvalRunPage;
|
|
12
12
|
export declare function listEvalResults(db: Database.Database, evalRunId: string): EvalResultWithGrades[];
|
|
13
|
+
/** Keys of the results one grader graded in a run, without their outputs. */
|
|
14
|
+
export declare function listGradedResultKeys(db: Database.Database, evalRunId: string, graderId: string): Array<{
|
|
15
|
+
caseId: string;
|
|
16
|
+
repeatIndex: number;
|
|
17
|
+
}>;
|
|
13
18
|
export declare function getEvalResultsForCase(db: Database.Database, evalRunId: string, caseId: string): EvalResultWithGrades[];
|
|
14
19
|
export declare function getEvalCases(db: Database.Database, evalRunId: string): EvalCaseRow[];
|
|
15
20
|
export declare function putEvalReview(db: Database.Database, evalRunId: string, caseId: string, repeatIndex: number, patch: EvalReviewPatch): EvalReviewRow;
|
|
@@ -335,6 +335,13 @@ export function listEvalResults(db, evalRunId) {
|
|
|
335
335
|
.all(evalRunId);
|
|
336
336
|
return attachGrades(db, rows.map(resultFromDb), evalRunId);
|
|
337
337
|
}
|
|
338
|
+
/** Keys of the results one grader graded in a run, without their outputs. */
|
|
339
|
+
export function listGradedResultKeys(db, evalRunId, graderId) {
|
|
340
|
+
return db
|
|
341
|
+
.prepare(`SELECT DISTINCT case_id, repeat_index FROM eval_grades
|
|
342
|
+
WHERE eval_run_id = ? AND grader_id = ? AND status = 'graded' ORDER BY case_id, repeat_index`)
|
|
343
|
+
.all(evalRunId, graderId).map((r) => ({ caseId: r.case_id, repeatIndex: r.repeat_index }));
|
|
344
|
+
}
|
|
338
345
|
export function getEvalResultsForCase(db, evalRunId, caseId) {
|
|
339
346
|
const rows = db
|
|
340
347
|
.prepare(`SELECT * FROM eval_results WHERE eval_run_id = ? AND case_id = ? ORDER BY repeat_index`)
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type Database from "better-sqlite3";
|
|
2
|
+
/**
|
|
3
|
+
* Human verdicts per judged grader on one result, and stored judge calibrations.
|
|
4
|
+
* Labels cascade with their result; calibrations belong to the eval, not a run. Additive.
|
|
5
|
+
*/
|
|
6
|
+
export declare const MIGRATION_018: {
|
|
7
|
+
version: number;
|
|
8
|
+
name: string;
|
|
9
|
+
minStageflowVersion: string;
|
|
10
|
+
up(db: Database.Database): void;
|
|
11
|
+
};
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { PACKAGE_VERSION } from "../../../package-meta.js";
|
|
2
|
+
/**
|
|
3
|
+
* Human verdicts per judged grader on one result, and stored judge calibrations.
|
|
4
|
+
* Labels cascade with their result; calibrations belong to the eval, not a run. Additive.
|
|
5
|
+
*/
|
|
6
|
+
export const MIGRATION_018 = {
|
|
7
|
+
version: 18,
|
|
8
|
+
name: "018_eval_grader_labels",
|
|
9
|
+
minStageflowVersion: PACKAGE_VERSION,
|
|
10
|
+
up(db) {
|
|
11
|
+
db.exec(`
|
|
12
|
+
CREATE TABLE eval_grader_labels (
|
|
13
|
+
eval_run_id TEXT NOT NULL,
|
|
14
|
+
case_id TEXT NOT NULL,
|
|
15
|
+
repeat_index INTEGER NOT NULL,
|
|
16
|
+
grader_id TEXT NOT NULL,
|
|
17
|
+
label TEXT NOT NULL CHECK (label IN ('pass','fail')),
|
|
18
|
+
note TEXT,
|
|
19
|
+
created_at TEXT NOT NULL,
|
|
20
|
+
updated_at TEXT NOT NULL,
|
|
21
|
+
PRIMARY KEY (eval_run_id, case_id, repeat_index, grader_id),
|
|
22
|
+
FOREIGN KEY (eval_run_id, case_id, repeat_index) REFERENCES eval_results (eval_run_id, case_id, repeat_index) ON DELETE CASCADE
|
|
23
|
+
);
|
|
24
|
+
|
|
25
|
+
CREATE TABLE eval_calibrations (
|
|
26
|
+
id TEXT PRIMARY KEY,
|
|
27
|
+
project_root TEXT NOT NULL,
|
|
28
|
+
eval_id TEXT NOT NULL,
|
|
29
|
+
grader_id TEXT NOT NULL,
|
|
30
|
+
grader_hash TEXT NOT NULL,
|
|
31
|
+
judge TEXT NOT NULL,
|
|
32
|
+
prompt_version INTEGER NOT NULL,
|
|
33
|
+
split TEXT NOT NULL CHECK (split IN ('all','dev','held_out')),
|
|
34
|
+
candidate INTEGER NOT NULL DEFAULT 0,
|
|
35
|
+
stats_json TEXT NOT NULL,
|
|
36
|
+
disagreements_json TEXT NOT NULL,
|
|
37
|
+
cost_usd REAL NOT NULL DEFAULT 0,
|
|
38
|
+
created_at TEXT NOT NULL
|
|
39
|
+
);
|
|
40
|
+
CREATE INDEX eval_calibrations_grader ON eval_calibrations (project_root, eval_id, grader_id, created_at);
|
|
41
|
+
`);
|
|
42
|
+
},
|
|
43
|
+
};
|
|
@@ -16,6 +16,7 @@ import { MIGRATION_014 } from "./014-agent-deliveries.js";
|
|
|
16
16
|
import { MIGRATION_015 } from "./015-evals.js";
|
|
17
17
|
import { MIGRATION_016 } from "./016-eval-subject-ref.js";
|
|
18
18
|
import { MIGRATION_017 } from "./017-eval-result-suggestions.js";
|
|
19
|
+
import { MIGRATION_018 } from "./018-eval-grader-labels.js";
|
|
19
20
|
const DEFAULT_MIGRATIONS = [
|
|
20
21
|
MIGRATION_001,
|
|
21
22
|
MIGRATION_002,
|
|
@@ -34,6 +35,7 @@ const DEFAULT_MIGRATIONS = [
|
|
|
34
35
|
MIGRATION_015,
|
|
35
36
|
MIGRATION_016,
|
|
36
37
|
MIGRATION_017,
|
|
38
|
+
MIGRATION_018,
|
|
37
39
|
];
|
|
38
40
|
export const CURRENT_SCHEMA_VERSION = DEFAULT_MIGRATIONS[DEFAULT_MIGRATIONS.length - 1].version;
|
|
39
41
|
function throwStoreSchemaTooNew(db, version, maxKnown) {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { access, lstat, mkdir, readdir, realpath } from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { globalCredentialRoot } from "../project/globalHome.js";
|
|
4
|
+
import { buildArtifactListedPath } from "../runtime/artifactPath.js";
|
|
4
5
|
import { STAGEFLOW_AGENT_AUTH_PATH_ENV } from "../runtime/credentialBinding.js";
|
|
5
6
|
import { SF_STAGE_WORKER } from "../runtime/stageWorkerProtocol.js";
|
|
6
7
|
export { stageDir } from "./paths.js";
|
|
@@ -186,7 +187,11 @@ export function resolveArtifactTarget(runWorkspaceDir, stageId, attempt, relativ
|
|
|
186
187
|
path.isAbsolute(relativeToArtifacts)) {
|
|
187
188
|
throw new Error("path escapes the stage artifacts directory");
|
|
188
189
|
}
|
|
189
|
-
const runRelativePath =
|
|
190
|
+
const runRelativePath = buildArtifactListedPath({
|
|
191
|
+
stageId,
|
|
192
|
+
attempt,
|
|
193
|
+
relative: relativeToArtifacts.split(path.sep).join("/"),
|
|
194
|
+
});
|
|
190
195
|
return {
|
|
191
196
|
absolutePath: resolved,
|
|
192
197
|
runRelativePath,
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export type ArtifactDiskFileType = "file" | "directory" | "symlink" | "other" | "missing";
|
|
2
|
+
export type ArtifactDiskPresence = {
|
|
3
|
+
file_type: ArtifactDiskFileType;
|
|
4
|
+
/** Lexical absolute path under root when the walk reached a leaf. */
|
|
5
|
+
absolutePath?: string;
|
|
6
|
+
/** From lstat when `file_type` is `file`. */
|
|
7
|
+
size_bytes?: number;
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* Closed-posture walk of `relativePath` under `rootDir`: lstat every segment,
|
|
11
|
+
* classify symlink / missing / file / directory / other. Does not follow
|
|
12
|
+
* symlinks. Escape attempts are `missing`. Non-ENOENT errors propagate.
|
|
13
|
+
*/
|
|
14
|
+
export declare function inspectArtifactUnderRoot(rootDir: string, relativePath: string): Promise<ArtifactDiskPresence>;
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { lstat } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
/**
|
|
4
|
+
* Closed-posture walk of `relativePath` under `rootDir`: lstat every segment,
|
|
5
|
+
* classify symlink / missing / file / directory / other. Does not follow
|
|
6
|
+
* symlinks. Escape attempts are `missing`. Non-ENOENT errors propagate.
|
|
7
|
+
*/
|
|
8
|
+
export async function inspectArtifactUnderRoot(rootDir, relativePath) {
|
|
9
|
+
const target = path.resolve(rootDir, relativePath);
|
|
10
|
+
const relative = path.relative(rootDir, target);
|
|
11
|
+
if (relative === "" ||
|
|
12
|
+
relative === ".." ||
|
|
13
|
+
relative.startsWith(`..${path.sep}`) ||
|
|
14
|
+
path.isAbsolute(relative)) {
|
|
15
|
+
return { file_type: "missing" };
|
|
16
|
+
}
|
|
17
|
+
try {
|
|
18
|
+
const segments = relative.split(path.sep).filter((segment) => segment !== "");
|
|
19
|
+
let current = rootDir;
|
|
20
|
+
let stats;
|
|
21
|
+
for (const segment of segments) {
|
|
22
|
+
current = path.join(current, segment);
|
|
23
|
+
stats = await lstat(current);
|
|
24
|
+
if (stats.isSymbolicLink()) {
|
|
25
|
+
return { file_type: "symlink", absolutePath: current };
|
|
26
|
+
}
|
|
27
|
+
if (current !== target && !stats.isDirectory()) {
|
|
28
|
+
return { file_type: "other", absolutePath: current };
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
if (stats === undefined) {
|
|
32
|
+
return { file_type: "missing" };
|
|
33
|
+
}
|
|
34
|
+
if (stats.isDirectory()) {
|
|
35
|
+
return { file_type: "directory", absolutePath: current };
|
|
36
|
+
}
|
|
37
|
+
if (!stats.isFile()) {
|
|
38
|
+
return { file_type: "other", absolutePath: current };
|
|
39
|
+
}
|
|
40
|
+
return {
|
|
41
|
+
file_type: "file",
|
|
42
|
+
absolutePath: current,
|
|
43
|
+
size_bytes: stats.size,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
catch (error) {
|
|
47
|
+
if (error.code === "ENOENT") {
|
|
48
|
+
return { file_type: "missing" };
|
|
49
|
+
}
|
|
50
|
+
throw error;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/** Parts of a run-relative artifact path under `stages/.../artifacts/`. */
|
|
2
|
+
export type ArtifactPathParts = {
|
|
3
|
+
stageId: string;
|
|
4
|
+
attempt: number;
|
|
5
|
+
relative: string;
|
|
6
|
+
};
|
|
7
|
+
export type ArtifactPathMode = {
|
|
8
|
+
kind: "grantable";
|
|
9
|
+
} | {
|
|
10
|
+
kind: "current_attempt";
|
|
11
|
+
stageId: string;
|
|
12
|
+
attempt: number;
|
|
13
|
+
};
|
|
14
|
+
export declare function attemptArtifactsPrefix(stageId: string, attempt: number): string;
|
|
15
|
+
export declare function buildArtifactListedPath(parts: ArtifactPathParts): string;
|
|
16
|
+
/**
|
|
17
|
+
* Accept a listed artifact path under a mode.
|
|
18
|
+
* - `grantable`: full run-relative path to a prior stage/attempt artifact file.
|
|
19
|
+
* - `current_attempt`: that attempt's run-relative prefix, or a bare path under
|
|
20
|
+
* its artifacts dir; foreign `stages/...` paths are rejected.
|
|
21
|
+
*/
|
|
22
|
+
export declare function acceptArtifactListedPath(listedPath: string, mode: ArtifactPathMode): ArtifactPathParts | undefined;
|
|
23
|
+
export declare function parseArtifactListedPath(listedPath: string): ArtifactPathParts | undefined;
|
|
24
|
+
export declare function isGrantableArtifactPath(listedPath: string): boolean;
|