specrails-core 5.3.0 → 5.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-runtime/capabilities.d.ts +27 -0
- package/dist/agent-runtime/capabilities.js +73 -0
- package/dist/agent-runtime/capabilities.js.map +1 -0
- package/dist/agent-runtime/cli-executor.d.ts +9 -0
- package/dist/agent-runtime/cli-executor.js +100 -30
- package/dist/agent-runtime/cli-executor.js.map +1 -1
- package/dist/agent-runtime/cli.d.ts +1 -0
- package/dist/agent-runtime/cli.js +59 -11
- package/dist/agent-runtime/cli.js.map +1 -1
- package/dist/agent-runtime/codex-schema.d.ts +4 -0
- package/dist/agent-runtime/codex-schema.js +31 -0
- package/dist/agent-runtime/codex-schema.js.map +1 -0
- package/dist/agent-runtime/config.d.ts +4 -0
- package/dist/agent-runtime/config.js +110 -9
- package/dist/agent-runtime/config.js.map +1 -1
- package/dist/agent-runtime/core-host.d.ts +9 -1
- package/dist/agent-runtime/core-host.js +55 -20
- package/dist/agent-runtime/core-host.js.map +1 -1
- package/dist/agent-runtime/efficiency-summary.d.ts +67 -0
- package/dist/agent-runtime/efficiency-summary.js +52 -0
- package/dist/agent-runtime/efficiency-summary.js.map +1 -0
- package/dist/agent-runtime/efficiency-types.d.ts +60 -0
- package/dist/agent-runtime/efficiency-types.js +8 -0
- package/dist/agent-runtime/efficiency-types.js.map +1 -0
- package/dist/agent-runtime/efficiency.d.ts +5 -0
- package/dist/agent-runtime/efficiency.js +35 -0
- package/dist/agent-runtime/efficiency.js.map +1 -0
- package/dist/agent-runtime/evaluation-corpus.d.ts +13 -0
- package/dist/agent-runtime/evaluation-corpus.js +8 -0
- package/dist/agent-runtime/evaluation-corpus.js.map +1 -0
- package/dist/agent-runtime/evaluation.d.ts +43 -0
- package/dist/agent-runtime/evaluation.js +171 -0
- package/dist/agent-runtime/evaluation.js.map +1 -0
- package/dist/agent-runtime/executor-types.d.ts +35 -2
- package/dist/agent-runtime/executor-types.js.map +1 -1
- package/dist/agent-runtime/executors.d.ts +1 -0
- package/dist/agent-runtime/executors.js +4 -0
- package/dist/agent-runtime/executors.js.map +1 -1
- package/dist/agent-runtime/graph/artifacts.d.ts +8 -22
- package/dist/agent-runtime/graph/artifacts.js +99 -73
- package/dist/agent-runtime/graph/artifacts.js.map +1 -1
- package/dist/agent-runtime/graph/nodes.d.ts +3 -0
- package/dist/agent-runtime/graph/nodes.js +73 -26
- package/dist/agent-runtime/graph/nodes.js.map +1 -1
- package/dist/agent-runtime/graph/roles.d.ts +4 -1
- package/dist/agent-runtime/graph/roles.js +100 -36
- package/dist/agent-runtime/graph/roles.js.map +1 -1
- package/dist/agent-runtime/graph/state.d.ts +8 -0
- package/dist/agent-runtime/graph/state.js.map +1 -1
- package/dist/agent-runtime/index.d.ts +7 -1
- package/dist/agent-runtime/index.js +7 -1
- package/dist/agent-runtime/index.js.map +1 -1
- package/dist/agent-runtime/kimi-acp.d.ts +4 -0
- package/dist/agent-runtime/kimi-acp.js +20 -9
- package/dist/agent-runtime/kimi-acp.js.map +1 -1
- package/dist/agent-runtime/openai-executor.d.ts +8 -0
- package/dist/agent-runtime/openai-executor.js +28 -10
- package/dist/agent-runtime/openai-executor.js.map +1 -1
- package/dist/agent-runtime/openspec-tool-server.d.ts +1 -0
- package/dist/agent-runtime/openspec-tool-server.js +37 -0
- package/dist/agent-runtime/openspec-tool-server.js.map +1 -0
- package/dist/agent-runtime/openspec.d.ts +123 -0
- package/dist/agent-runtime/openspec.js +263 -0
- package/dist/agent-runtime/openspec.js.map +1 -0
- package/dist/agent-runtime/prompts.d.ts +5 -1
- package/dist/agent-runtime/prompts.js +59 -40
- package/dist/agent-runtime/prompts.js.map +1 -1
- package/dist/agent-runtime/provider-diagnostic.d.ts +2 -0
- package/dist/agent-runtime/provider-diagnostic.js +25 -0
- package/dist/agent-runtime/provider-diagnostic.js.map +1 -0
- package/dist/agent-runtime/repository-context.d.ts +25 -0
- package/dist/agent-runtime/repository-context.js +100 -0
- package/dist/agent-runtime/repository-context.js.map +1 -0
- package/dist/agent-runtime/review-context.d.ts +14 -0
- package/dist/agent-runtime/review-context.js +41 -0
- package/dist/agent-runtime/review-context.js.map +1 -0
- package/dist/agent-runtime/role-routing.d.ts +10 -0
- package/dist/agent-runtime/role-routing.js +29 -0
- package/dist/agent-runtime/role-routing.js.map +1 -0
- package/dist/agent-runtime/role-state.d.ts +19 -0
- package/dist/agent-runtime/role-state.js +24 -0
- package/dist/agent-runtime/role-state.js.map +1 -0
- package/dist/agent-runtime/runtime-identity.d.ts +11 -0
- package/dist/agent-runtime/runtime-identity.js +33 -0
- package/dist/agent-runtime/runtime-identity.js.map +1 -0
- package/dist/agent-runtime/tool-event.d.ts +3 -0
- package/dist/agent-runtime/tool-event.js +24 -0
- package/dist/agent-runtime/tool-event.js.map +1 -0
- package/dist/agent-runtime/verification-plan.d.ts +55 -0
- package/dist/agent-runtime/verification-plan.js +206 -0
- package/dist/agent-runtime/verification-plan.js.map +1 -0
- package/dist/agent-runtime/workflow-types.d.ts +17 -1
- package/dist/agent-runtime/workflow.js +28 -1
- package/dist/agent-runtime/workflow.js.map +1 -1
- package/dist/agent-runtime/workspace-tools.d.ts +4 -0
- package/dist/agent-runtime/workspace-tools.js +182 -20
- package/dist/agent-runtime/workspace-tools.js.map +1 -1
- package/dist/installer/phases/scaffold.js +39 -109
- package/dist/installer/phases/scaffold.js.map +1 -1
- package/dist/installer/runtime/pipeline-state.d.ts +142 -1
- package/dist/installer/runtime/pipeline-state.js +477 -28
- package/dist/installer/runtime/pipeline-state.js.map +1 -1
- package/docs/agent-runtime-efficiency.md +65 -0
- package/docs/agent-runtime.md +86 -15
- package/integration-contract.json +2 -1
- package/package.json +5 -2
- package/schemas/agent-runtime.schema.json +25 -3
- package/schemas/fixtures/runtime-efficiency-summary.v1.json +790 -0
- package/templates/codex-skills/batch-implement/SKILL.md +33 -58
- package/templates/codex-skills/implement/SKILL.md +21 -124
- package/templates/codex-skills/retry/SKILL.md +8 -34
- package/templates/commands/specrails/batch-implement.md +21 -16
- package/templates/commands/specrails/implement.md +17 -276
- package/templates/commands/specrails/retry.md +6 -34
- package/templates/gemini-commands/batch-implement.toml +34 -28
- package/templates/gemini-commands/implement.toml +34 -55
- package/templates/gemini-commands/retry.toml +10 -16
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
import { fingerprint } from './durable-store.js';
|
|
2
|
+
import { prepareOpenSpec, roleOpenSpecContext } from './openspec.js';
|
|
1
3
|
import { existsSync, readFileSync } from 'node:fs';
|
|
2
4
|
import path from 'node:path';
|
|
3
|
-
import { initializePipeline, inspectPipeline, pipelineStateDirectory, validatePipelineContext, validateVerificationRequest, } from '../installer/runtime/pipeline-state.js';
|
|
4
|
-
import {
|
|
5
|
+
import { fingerprintCandidate, initializePipeline, inspectPipeline, pipelineStateDirectory, validatePipelineContext, validateVerificationRequest, } from '../installer/runtime/pipeline-state.js';
|
|
6
|
+
import { normalizeRuntimeConfig } from './config.js';
|
|
5
7
|
import { createExecutorRegistry } from './executors.js';
|
|
6
8
|
import { archive, child, journal, parseAgentObject, SLUG } from './graph/artifacts.js';
|
|
7
9
|
import { coreNodes } from './graph/nodes.js';
|
|
@@ -9,22 +11,20 @@ import { resolveReviewPolicy } from './graph/review-policy.js';
|
|
|
9
11
|
import { createRoleInvoker } from './graph/roles.js';
|
|
10
12
|
import { CORE_NODE_ORDER, CoreState } from './graph/state.js';
|
|
11
13
|
import { ROLE_INSTRUCTIONS_VERSION } from './prompts.js';
|
|
14
|
+
import { assertEffortSupported } from './capabilities.js';
|
|
15
|
+
import { runtimePackageIntegrity } from './runtime-identity.js';
|
|
12
16
|
import { readWorkflowState, runWorkflow } from './workflow.js';
|
|
13
17
|
export const RUNTIME_API_VERSION = 1;
|
|
14
|
-
export const CORE_WORKFLOW_VERSION = '
|
|
18
|
+
export const CORE_WORKFLOW_VERSION = '5';
|
|
15
19
|
export const CORE_PACKAGE_VERSION = JSON.parse(readFileSync(new URL('../../package.json', import.meta.url), 'utf8')).version;
|
|
20
|
+
export function coreRuntimeIdentity() {
|
|
21
|
+
return { packageVersion: CORE_PACKAGE_VERSION, workflowVersion: CORE_WORKFLOW_VERSION, instructionsVersion: String(ROLE_INSTRUCTIONS_VERSION), packageIntegrity: runtimePackageIntegrity(), apiVersion: 1 };
|
|
22
|
+
}
|
|
16
23
|
export { parseAgentObject };
|
|
17
|
-
/**
|
|
18
|
-
|
|
19
|
-
* with bounded corrections routed back to the developer, an autonomous investigation pass
|
|
20
|
-
* before a low-confidence design asks the requester, real verification receipts, acceptance
|
|
21
|
-
* evidence bound to the exact candidate, and an optional approval before archive.
|
|
22
|
-
*/
|
|
23
|
-
export async function runCoreWorkflow(options) {
|
|
24
|
+
/** Read-only admission: validate scope, limits and actual effort support before freezing a request. */
|
|
25
|
+
export async function preflightCoreWorkflow(options) {
|
|
24
26
|
const context = validatePipelineContext(options.context);
|
|
25
|
-
const config =
|
|
26
|
-
if (!config.enabled)
|
|
27
|
-
throw new Error('Programmatic agent runtime is disabled');
|
|
27
|
+
const config = normalizeRuntimeConfig(options.config, { registeredProviderIds: options.registry?.ids() });
|
|
28
28
|
if (context.ownership.git !== 'host')
|
|
29
29
|
throw new Error('Programmatic runtime requires host-owned delivery; Core shipping is not implemented by this workflow');
|
|
30
30
|
if (!SLUG.test(options.change) || options.change.length > 100)
|
|
@@ -36,11 +36,28 @@ export async function runCoreWorkflow(options) {
|
|
|
36
36
|
if (config.verification.length)
|
|
37
37
|
validateVerificationRequest(context, { kind: 'scoped', commands: config.verification });
|
|
38
38
|
const registry = options.registry ?? createExecutorRegistry(config);
|
|
39
|
-
for (const selected of Object.values(config.agents))
|
|
39
|
+
for (const selected of Object.values(config.agents)) {
|
|
40
40
|
registry.validateLimits(selected.provider, config.limits ?? {});
|
|
41
|
-
|
|
41
|
+
for (const tier of [selected, ...(selected.escalation ? [selected.escalation] : [])]) {
|
|
42
|
+
if (tier.effort !== undefined)
|
|
43
|
+
assertEffortSupported(tier, await registry.capabilities(selected.provider, tier.model));
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return { context, config, registry };
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Runs the Core implementation graph: architect → developer → verify → reviewer → archive,
|
|
50
|
+
* with bounded corrections routed back to the developer, an autonomous investigation pass
|
|
51
|
+
* before a low-confidence design asks the requester, real verification receipts, acceptance
|
|
52
|
+
* evidence bound to the exact candidate, and an optional approval before archive.
|
|
53
|
+
*/
|
|
54
|
+
export async function runCoreWorkflow(options) {
|
|
55
|
+
const { context, config, registry } = await preflightCoreWorkflow(options);
|
|
42
56
|
const directory = path.join(pipelineStateDirectory(context), 'agent-workflow');
|
|
43
57
|
const previous = await readWorkflowState(directory, context.runId);
|
|
58
|
+
if (previous && previous.workflowVersion !== CORE_WORKFLOW_VERSION)
|
|
59
|
+
throw new Error('This saved run requires its original runtime package. Restore the retained original Core runtime; its checkpoint has not been migrated.');
|
|
60
|
+
const state = initializePipeline(context, options.change);
|
|
44
61
|
if (!previous && existsSync(path.join(context.artifactRoot, 'openspec/changes', options.change))) {
|
|
45
62
|
throw new Error('Change already exists without a programmatic checkpoint; use a new change name');
|
|
46
63
|
}
|
|
@@ -50,16 +67,30 @@ export async function runCoreWorkflow(options) {
|
|
|
50
67
|
throw new Error('Archived run evidence changed; start a new run for the changed candidate or environment');
|
|
51
68
|
}
|
|
52
69
|
}
|
|
70
|
+
const prepared = prepareOpenSpec(context.artifactRoot, options.change, directory);
|
|
71
|
+
const openspec = Object.fromEntries(['architect', 'developer', 'reviewer'].map(role => {
|
|
72
|
+
const provider = config.providers.find(item => item.id === config.agents[role].provider);
|
|
73
|
+
return [role, roleOpenSpecContext(prepared, context.artifactRoot, options.change, directory, role, provider?.kind === 'cli' ? provider.cli : 'claude')];
|
|
74
|
+
}));
|
|
75
|
+
for (const role of ['architect', 'developer', 'reviewer'])
|
|
76
|
+
await registry.get(config.agents[role].provider).validateOpenSpec?.(openspec[role]);
|
|
53
77
|
const attempts = config.limits?.maxAttempts ?? 3;
|
|
54
78
|
const note = (role, text) => { try {
|
|
55
79
|
options.onAgentEvent?.(role, { kind: 'text', text });
|
|
56
80
|
}
|
|
57
81
|
catch { /* Observer cannot replay agent effects. */ } };
|
|
58
|
-
const invoke = createRoleInvoker({ context, config, registry, onAgentEvent: options.onAgentEvent });
|
|
59
|
-
|
|
82
|
+
const invoke = createRoleInvoker({ context, config, registry, openspec, onAgentEvent: options.onAgentEvent });
|
|
83
|
+
let grantedArchiveScope;
|
|
84
|
+
const archiveConsent = () => {
|
|
85
|
+
const pipeline = journal(context);
|
|
86
|
+
return fingerprint({ candidate: fingerprintCandidate(pipeline), plan: pipeline.verificationPlan?.hash ?? null,
|
|
87
|
+
criteria: pipeline.acceptance?.criteria.map(({ evidence: _evidence, ...criterion }) => criterion) ?? null,
|
|
88
|
+
checks: pipeline.acceptance?.checks.map(({ evidence: _evidence, ...check }) => check) ?? null });
|
|
89
|
+
};
|
|
90
|
+
const nodes = coreNodes({ archiveApproved: () => grantedArchiveScope !== undefined && grantedArchiveScope === archiveConsent(), context, config, openspec, change: options.change, attempts, policy: resolveReviewPolicy(config), invoke, note, onVerificationOutput: options.onVerificationOutput });
|
|
60
91
|
return runWorkflow({
|
|
61
92
|
directory, runId: context.runId,
|
|
62
|
-
input: JSON.parse(JSON.stringify({ context, config, change: options.change, coreVersion: CORE_PACKAGE_VERSION, instructionsVersion: ROLE_INSTRUCTIONS_VERSION })),
|
|
93
|
+
input: JSON.parse(JSON.stringify({ context, config, change: options.change, coreVersion: CORE_PACKAGE_VERSION, runtimeIdentity: coreRuntimeIdentity(), instructionsVersion: ROLE_INSTRUCTIONS_VERSION, openspec: prepared.identity })),
|
|
63
94
|
resume: options.resume, signal: options.signal, approve: options.approve, answer: options.answer, recoverInterrupted: options.recoverInterrupted, invalidate: options.invalidate,
|
|
64
95
|
budget: { maxCostUsd: config.limits?.maxCostUsd, maxTokens: config.limits?.maxTokens, maxDurationMs: config.limits?.timeoutMs },
|
|
65
96
|
onEvent: options.onEvent, onSpan: options.onSpan,
|
|
@@ -74,13 +105,17 @@ export async function runCoreWorkflow(options) {
|
|
|
74
105
|
&& options.recoverInterrupted?.includes('archive')
|
|
75
106
|
&& journal(context).phases.archive.status === 'running'
|
|
76
107
|
&& !existsSync(child(context.artifactRoot, 'openspec/changes/' + options.change))) {
|
|
77
|
-
archive(context, options.change);
|
|
108
|
+
await archive(context, options.change);
|
|
78
109
|
}
|
|
79
110
|
const inspection = inspectPipeline(context);
|
|
111
|
+
if (stepId === 'architect' && checkpoint.pendingApproval?.stepId === 'archive' && options.approve?.includes('archive') && inspection.verification.valid && inspection.acceptance.valid && inspection.phases.reviewer.status === 'done')
|
|
112
|
+
grantedArchiveScope = archiveConsent();
|
|
80
113
|
if (stepId === 'architect')
|
|
81
114
|
return inspection.phases.architect.status === 'done' && inspection.resumePhase !== 'architect';
|
|
115
|
+
if (stepId === 'verify' && checkpoint.status !== 'succeeded' && inspection.phases.archive.status !== 'done' && !(checkpoint.pendingApproval?.stepId === 'archive' && !options.approve?.includes('archive') && !checkpoint.pendingApproval.grantedAt))
|
|
116
|
+
return false;
|
|
82
117
|
if (stepId === 'verify')
|
|
83
|
-
return inspection.verification.valid;
|
|
118
|
+
return _record.output?.valid !== false && inspection.verification.valid;
|
|
84
119
|
if (stepId === 'reviewer')
|
|
85
120
|
return inspection.phases.reviewer.status === 'done' && inspection.verification.valid && !['architect', 'developer', 'reviewer'].includes(inspection.resumePhase ?? '');
|
|
86
121
|
if (stepId === 'archive')
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"core-host.js","sourceRoot":"","sources":["../../src/agent-runtime/core-host.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAClD,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EACL,kBAAkB,EAAE,eAAe,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,2BAA2B,
|
|
1
|
+
{"version":3,"file":"core-host.js","sourceRoot":"","sources":["../../src/agent-runtime/core-host.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAA;AAChD,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAA;AACpE,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAClD,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EACL,oBAAoB,EAAE,kBAAkB,EAAE,eAAe,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,2BAA2B,GAExI,MAAM,wCAAwC,CAAA;AAC/C,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAA;AACpD,OAAO,EAAE,sBAAsB,EAAyB,MAAM,gBAAgB,CAAA;AAE9E,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAA;AACtF,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAA;AAC9D,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AACpD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAsB,MAAM,kBAAkB,CAAA;AACjF,OAAO,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAA;AACxD,OAAO,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAA;AACzD,OAAO,EAAE,uBAAuB,EAAwB,MAAM,uBAAuB,CAAA;AACrF,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAG9D,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAA;AACpC,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,CAAA;AACxC,MAAM,CAAC,MAAM,oBAAoB,GAAI,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,oBAAoB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAAyB,CAAC,OAAO,CAAA;AACrJ,MAAM,UAAU,mBAAmB;IACjC,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,eAAe,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,CAAC,yBAAyB,CAAC,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,CAAA;AAC7M,CAAC;AACD,OAAO,EAAE,gBAAgB,EAAE,CAAA;AAoB3B,uGAAuG;AACvG,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,OAAgF;IAC1H,MAAM,OAAO,GAAG,uBAAuB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IACxD,MAAM,MAAM,GAAG,sBAAsB,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,qBAAqB,EAAE,OAAO,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,CAAC,CAAA;IACzG,IAAI,OAAO,CAAC,SAAS,CAAC,GAAG,KAAK,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,sGAAsG,CAAC,CAAA;IAC7J,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAA;IAC9G,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAA;IACxG,gFAAgF;IAChF,6EAA6E;IAC7E,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM;QAAE,2BAA2B,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC,CAAA;IACvH,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,sBAAsB,CAAC,MAAM,CAAC,CAAA;IACnE,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QACpD,QAAQ,CAAC,cAAc,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAA;QAC/D,KAAK,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YACrF,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS;gBAAE,qBAAqB,CAAC,IAAI,EAAE,MAAM,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;QACxH,CAAC;IACH,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;AACtC,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAA4B;IAChE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,qBAAqB,CAAC,OAAO,CAAC,CAAA;IAC1E,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,EAAE,gBAAgB,CAAC,CAAA;IAC9E,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;IAClE,IAAI,QAAQ,IAAI,QAAQ,CAAC,eAAe,KAAK,qBAAqB;QAAE,MAAM,IAAI,KAAK,CAAC,yIAAyI,CAAC,CAAA;IAC9N,MAAM,KAAK,GAAG,kBAAkB,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;IACzD,IAAI,CAAC,QAAQ,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,kBAAkB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;QACjG,MAAM,IAAI,KAAK,CAAC,gFAAgF,CAAC,CAAA;IACnG,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC,CAAA;QACxC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC;YAC1G,MAAM,IAAI,KAAK,CAAC,yFAAyF,CAAC,CAAA;QAC5G,CAAC;IACH,CAAC;IACD,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;IACjF,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAE,CAAC,WAAW,EAAE,WAAW,EAAE,UAAU,CAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QAC/F,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAA;QACxF,OAAO,CAAC,IAAI,EAAE,mBAAmB,CAAC,QAAQ,EAAE,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAA;IACzJ,CAAC,CAAC,CAA8D,CAAA;IAChE,KAAK,MAAM,IAAI,IAAI,CAAC,WAAW,EAAE,WAAW,EAAE,UAAU,CAAU;QAAE,MAAM,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,gBAAgB,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;IACvJ,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,EAAE,WAAW,IAAI,CAAC,CAAA;IAChD,MAAM,IAAI,GAAG,CAAC,IAAe,EAAE,IAAY,EAAQ,EAAE,GAAG,IAAI,CAAC;QAAC,OAAO,CAAC,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;IAAC,CAAC;IAAC,MAAM,CAAC,CAAC,2CAA2C,CAAC,CAAC,CAAC,CAAC,CAAA;IAC5K,MAAM,MAAM,GAAG,iBAAiB,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC,CAAA;IAC7G,IAAI,mBAAuC,CAAA;IAC3C,MAAM,cAAc,GAAG,GAAW,EAAE;QAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;QACjC,OAAO,WAAW,CAAC,EAAE,SAAS,EAAE,oBAAoB,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,gBAAgB,EAAE,IAAI,IAAI,IAAI;YAC3G,QAAQ,EAAE,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,SAAS,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,IAAI;YACzG,MAAM,EAAE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC,CAAA;IACpG,CAAC,CAAA;IACD,MAAM,KAAK,GAAG,SAAS,CAAC,EAAE,eAAe,EAAE,GAAG,EAAE,CAAC,mBAAmB,KAAK,SAAS,IAAI,mBAAmB,KAAK,cAAc,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,mBAAmB,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAAA;IACrS,OAAO,WAAW,CAAgB;QAChC,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK;QAC/B,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,WAAW,EAAE,oBAAoB,EAAE,eAAe,EAAE,mBAAmB,EAAE,EAAE,mBAAmB,EAAE,yBAAyB,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAc;QACnP,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU;QAChL,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE;QAC/H,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM;QAChD,iBAAiB,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,EAAE;YACvD,2EAA2E;YAC3E,2EAA2E;YAC3E,wEAAwE;YACxE,0EAA0E;YAC1E,wEAAwE;YACxE,IAAI,MAAM,KAAK,WAAW,IAAI,UAAU,CAAC,QAAQ,KAAK,SAAS;mBAC1D,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,KAAK,aAAa;mBAClD,OAAO,CAAC,kBAAkB,EAAE,QAAQ,CAAC,SAAS,CAAC;mBAC/C,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS;mBACpD,CAAC,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;gBACpF,MAAM,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;YACxC,CAAC;YACD,MAAM,UAAU,GAAG,eAAe,CAAC,OAAO,CAAC,CAAA;YAC3C,IAAI,MAAM,KAAK,WAAW,IAAI,UAAU,CAAC,eAAe,EAAE,MAAM,KAAK,SAAS,IAAI,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,SAAS,CAAC,IAAI,UAAU,CAAC,YAAY,CAAC,KAAK,IAAI,UAAU,CAAC,UAAU,CAAC,KAAK,IAAI,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,MAAM;gBAAE,mBAAmB,GAAG,cAAc,EAAE,CAAA;YAC9Q,IAAI,MAAM,KAAK,WAAW;gBAAE,OAAO,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,KAAK,MAAM,IAAI,UAAU,CAAC,WAAW,KAAK,WAAW,CAAA;YAC1H,IAAI,MAAM,KAAK,QAAQ,IAAI,UAAU,CAAC,MAAM,KAAK,WAAW,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,CAAC,UAAU,CAAC,eAAe,EAAE,MAAM,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,SAAS,CAAC;gBAAE,OAAO,KAAK,CAAA;YAClQ,IAAI,MAAM,KAAK,QAAQ;gBAAE,OAAQ,OAAO,CAAC,MAA0C,EAAE,KAAK,KAAK,KAAK,IAAI,UAAU,CAAC,YAAY,CAAC,KAAK,CAAA;YACrI,IAAI,MAAM,KAAK,UAAU;gBAAE,OAAO,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,MAAM,IAAI,UAAU,CAAC,YAAY,CAAC,KAAK,IAAI,CAAC,CAAC,WAAW,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,IAAI,EAAE,CAAC,CAAA;YACjM,IAAI,MAAM,KAAK,SAAS;gBAAE,OAAO,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,MAAM,CAAA;YAC5E,OAAO,IAAI,CAAA;QACb,CAAC;QACD,QAAQ,EAAE;YACR,EAAE,EAAE,0BAA0B,EAAE,OAAO,EAAE,qBAAqB,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW;YACrG,cAAc,EAAE,QAAQ,GAAG,CAAC,GAAG,CAAC;YAChC,KAAK,EAAE,MAAM,CAAC,WAAW,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;SACtE;KACF,CAAC,CAAA;AACJ,CAAC"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { type PipelineContext, type PipelineCompletion } from '../installer/runtime/pipeline-state.js';
|
|
2
|
+
import type { RuntimeConfig } from './executor-types.js';
|
|
3
|
+
import type { WorkflowState } from './workflow-types.js';
|
|
4
|
+
/** Compact derived view. Durable invocation/evidence IDs remain the accounting authority. */
|
|
5
|
+
export declare function efficiencySummary(state: WorkflowState, context: PipelineContext, config: RuntimeConfig | undefined, completion?: PipelineCompletion, identity?: {
|
|
6
|
+
planHash?: string;
|
|
7
|
+
candidateHash: string;
|
|
8
|
+
verification?: {
|
|
9
|
+
receipt?: {
|
|
10
|
+
commands: Array<{
|
|
11
|
+
evidenceId?: string;
|
|
12
|
+
}>;
|
|
13
|
+
notRunEvidenceIds?: string[];
|
|
14
|
+
};
|
|
15
|
+
};
|
|
16
|
+
}): {
|
|
17
|
+
schemaVersion: 1;
|
|
18
|
+
currentEvidenceIds: string[];
|
|
19
|
+
planHash: string | null;
|
|
20
|
+
candidateHash: string | null;
|
|
21
|
+
runId: string;
|
|
22
|
+
workflowVersion: string;
|
|
23
|
+
technicalAcceptance: string;
|
|
24
|
+
archive: import("../installer/runtime/pipeline-state.js").PhaseStatus;
|
|
25
|
+
delivery: "pending" | "complete" | "pending-host";
|
|
26
|
+
roles: {
|
|
27
|
+
role: "architect" | "developer" | "reviewer";
|
|
28
|
+
provider: string | null;
|
|
29
|
+
model: string | null;
|
|
30
|
+
effort: string | null;
|
|
31
|
+
observedModel: null;
|
|
32
|
+
observedEffort: null;
|
|
33
|
+
origin: string;
|
|
34
|
+
tier: "base" | "escalation";
|
|
35
|
+
}[];
|
|
36
|
+
invocations: {
|
|
37
|
+
byKind: {
|
|
38
|
+
[k: string]: number;
|
|
39
|
+
};
|
|
40
|
+
total: number;
|
|
41
|
+
complete: boolean;
|
|
42
|
+
promptBytes: number | null;
|
|
43
|
+
contextBytes: number | null;
|
|
44
|
+
handoffBytes: number | null;
|
|
45
|
+
fullContexts: number;
|
|
46
|
+
incrementalContexts: number;
|
|
47
|
+
};
|
|
48
|
+
escalations: {
|
|
49
|
+
role: string;
|
|
50
|
+
attemptId: string;
|
|
51
|
+
provider: string;
|
|
52
|
+
model: string | null;
|
|
53
|
+
effort: string | null;
|
|
54
|
+
reason: string | null;
|
|
55
|
+
}[];
|
|
56
|
+
escalationsTruncated: boolean;
|
|
57
|
+
checks: {
|
|
58
|
+
invalidated: number | null;
|
|
59
|
+
available: boolean;
|
|
60
|
+
complete: boolean;
|
|
61
|
+
executed: number | null;
|
|
62
|
+
reused: number | null;
|
|
63
|
+
notRun: number | null;
|
|
64
|
+
durationMs: number | null;
|
|
65
|
+
};
|
|
66
|
+
metrics: import("./efficiency-types.js").RuntimeEfficiency;
|
|
67
|
+
};
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { readVerificationEvidence } from '../installer/runtime/pipeline-state.js';
|
|
2
|
+
import { runtimeEfficiency } from './efficiency.js';
|
|
3
|
+
/** Compact derived view. Durable invocation/evidence IDs remain the accounting authority. */
|
|
4
|
+
export function efficiencySummary(state, context, config, completion, identity) {
|
|
5
|
+
const invocations = state.history.flatMap(attempt => [...(attempt.invocations ?? []), ...(attempt.pendingInvocations ?? [])].map(call => ({ ...call, role: attempt.stepId, attemptId: attempt.id })));
|
|
6
|
+
const kinds = ['initial', 'correction', 'repair', 'deepen', 'session-fallback'];
|
|
7
|
+
const byKind = Object.fromEntries(kinds.map(kind => [kind, invocations.filter(call => call.kind === kind).length]));
|
|
8
|
+
const measured = state.history.every(attempt => !attempt.pendingInvocations?.length) && state.history.every(attempt => !['architect', 'developer', 'reviewer'].includes(attempt.stepId) || attempt.invocations !== undefined) && invocations.every(call => call.kind !== undefined && call.promptBytes !== undefined);
|
|
9
|
+
const bytes = (key) => invocations.every(call => call[key] !== undefined) ? invocations.reduce((sum, call) => sum + call[key], 0) : null;
|
|
10
|
+
const checkIds = new Set();
|
|
11
|
+
let checks = [], evidenceAvailable = true;
|
|
12
|
+
try {
|
|
13
|
+
let cursor;
|
|
14
|
+
do {
|
|
15
|
+
const page = readVerificationEvidence(context, { limit: 100, ...(cursor ? { cursor } : {}) });
|
|
16
|
+
if (!page.available) {
|
|
17
|
+
evidenceAvailable = false;
|
|
18
|
+
break;
|
|
19
|
+
}
|
|
20
|
+
for (const row of page.items ?? [])
|
|
21
|
+
if (typeof row.id === 'string' && !checkIds.has(row.id)) {
|
|
22
|
+
checkIds.add(row.id);
|
|
23
|
+
checks.push(row);
|
|
24
|
+
}
|
|
25
|
+
if (checkIds.size > 10000)
|
|
26
|
+
throw new Error('Evidence summary limit');
|
|
27
|
+
cursor = 'nextCursor' in page ? page.nextCursor : undefined;
|
|
28
|
+
} while (cursor);
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
checks = [];
|
|
32
|
+
evidenceAvailable = false;
|
|
33
|
+
}
|
|
34
|
+
const checksComplete = evidenceAvailable && checks.every(check => check.status !== 'interrupted' && typeof check.durationMs === 'number');
|
|
35
|
+
const escalatedRoles = new Set();
|
|
36
|
+
const escalations = invocations.filter(call => { if (call.tier !== 'escalation' || escalatedRoles.has(call.role))
|
|
37
|
+
return false; escalatedRoles.add(call.role); return true; }).map(call => ({ role: call.role, attemptId: call.attemptId, provider: call.provider, model: call.model ?? null, effort: call.requestedEffort ?? null, reason: call.routeReason ?? null }));
|
|
38
|
+
return {
|
|
39
|
+
schemaVersion: 1, currentEvidenceIds: [...(identity?.verification?.receipt?.commands.flatMap(command => command.evidenceId ? [command.evidenceId] : []) ?? []), ...(identity?.verification?.receipt?.notRunEvidenceIds ?? [])].slice(0, 100), planHash: identity?.planHash ?? null, candidateHash: identity?.candidateHash ?? null, runId: state.runId, workflowVersion: state.workflowVersion,
|
|
40
|
+
technicalAcceptance: completion?.validation === 'verified' ? 'validated' : completion?.validation === 'with-exceptions' ? 'with-exceptions' : completion?.validation === 'blocked' ? 'blocked' : 'pending',
|
|
41
|
+
archive: completion?.archive ?? 'pending', delivery: completion?.delivery ?? 'pending',
|
|
42
|
+
roles: ['architect', 'developer', 'reviewer'].map(role => {
|
|
43
|
+
const selected = config?.agents[role], latest = invocations.filter(call => call.role === role).at(-1);
|
|
44
|
+
return { role, provider: latest?.provider ?? selected?.provider ?? null, model: latest?.model ?? selected?.model ?? null, effort: latest ? latest.requestedEffort ?? null : selected?.effort ?? null, observedModel: null, observedEffort: null, origin: 'frozen-runtime-config', tier: latest?.tier ?? 'base' };
|
|
45
|
+
}),
|
|
46
|
+
invocations: { byKind, total: invocations.length, complete: measured, promptBytes: bytes('promptBytes'), contextBytes: bytes('contextBytes'), handoffBytes: bytes('handoffBytes'), fullContexts: invocations.filter(call => call.contextMode === 'full').length, incrementalContexts: invocations.filter(call => call.contextMode === 'incremental').length },
|
|
47
|
+
escalations: escalations.slice(-16), escalationsTruncated: escalations.length > 16,
|
|
48
|
+
checks: { invalidated: identity && evidenceAvailable ? checks.filter(check => check.planHash !== identity.planHash || check.candidateHash !== identity.candidateHash || check.reuseReason === 'snapshot-inputs-changed-during-verification').length : null, available: evidenceAvailable, complete: checksComplete, executed: checksComplete ? checks.filter(check => check.disposition === 'executed').length : null, reused: evidenceAvailable ? checks.filter(check => check.disposition === 'reused').length : null, notRun: evidenceAvailable ? checks.filter(check => check.disposition === 'not-run').length : null, durationMs: checksComplete ? checks.reduce((sum, check) => sum + (typeof check.durationMs === 'number' ? check.durationMs : 0), 0) : null },
|
|
49
|
+
metrics: runtimeEfficiency(state),
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=efficiency-summary.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"efficiency-summary.js","sourceRoot":"","sources":["../../src/agent-runtime/efficiency-summary.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,EAAiD,MAAM,wCAAwC,CAAA;AAGhI,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAEnD,6FAA6F;AAC7F,MAAM,UAAU,iBAAiB,CAAC,KAAoB,EAAE,OAAwB,EAAE,MAAiC,EAAE,UAA+B,EAAE,QAAgK;IACpT,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAA;IACrM,MAAM,KAAK,GAAG,CAAC,SAAS,EAAE,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,kBAAkB,CAAU,CAAA;IACxF,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;IACnH,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,CAAC,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,CAAA;IACrT,MAAM,KAAK,GAAG,CAAC,GAAoD,EAAE,EAAE,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAC1L,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA;IAClC,IAAI,MAAM,GAAmC,EAAE,EAAE,iBAAiB,GAAG,IAAI,CAAA;IACzE,IAAI,CAAC;QACH,IAAI,MAA0B,CAAA;QAC9B,GAAG,CAAC;YACF,MAAM,IAAI,GAAG,wBAAwB,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAA;YAC7F,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;gBAAC,iBAAiB,GAAG,KAAK,CAAC;gBAAC,MAAK;YAAC,CAAC;YACzD,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,IAAI,EAAE;gBAAE,IAAI,OAAO,GAAG,CAAC,EAAE,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;oBAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBAAC,CAAC;YACvI,IAAI,QAAQ,CAAC,IAAI,GAAG,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAA;YACpE,MAAM,GAAG,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAA;QAC7D,CAAC,QAAQ,MAAM,EAAC;IAClB,CAAC;IAAC,MAAM,CAAC;QAAC,MAAM,GAAG,EAAE,CAAC;QAAC,iBAAiB,GAAG,KAAK,CAAA;IAAC,CAAC;IAClD,MAAM,cAAc,GAAG,iBAAiB,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,aAAa,IAAI,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAA;IACzI,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAA;IACxC,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,IAAI,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,IAAI,CAAA,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,eAAe,IAAI,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,IAAI,IAAI,EAAE,CAAC,CAAC,CAAA;IACvW,OAAO;QACL,aAAa,EAAE,CAAU,EAAE,kBAAkB,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,iBAAiB,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,IAAI,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,aAAa,IAAI,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,eAAe,EAAE,KAAK,CAAC,eAAe;QACvY,mBAAmB,EAAE,UAAU,EAAE,UAAU,KAAK,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,EAAE,UAAU,KAAK,iBAAiB,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,UAAU,EAAE,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;QAC1M,OAAO,EAAE,UAAU,EAAE,OAAO,IAAI,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,IAAI,SAAS;QACtF,KAAK,EAAG,CAAC,WAAW,EAAE,WAAW,EAAE,UAAU,CAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;YAClE,MAAM,QAAQ,GAAG,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;YACrG,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,IAAI,QAAQ,EAAE,QAAQ,IAAI,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,IAAI,QAAQ,EAAE,KAAK,IAAI,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,IAAI,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,MAAM,IAAI,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,EAAE,uBAAuB,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,IAAI,MAAM,EAAE,CAAA;QAClT,CAAC,CAAC;QACF,WAAW,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,aAAa,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,cAAc,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,cAAc,CAAC,EAAE,YAAY,EAAE,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,KAAK,MAAM,CAAC,CAAC,MAAM,EAAE,mBAAmB,EAAE,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,KAAK,aAAa,CAAC,CAAC,MAAM,EAAE;QAC7V,WAAW,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,oBAAoB,EAAE,WAAW,CAAC,MAAM,GAAG,EAAE;QAClF,MAAM,EAAE,EAAE,WAAW,EAAE,QAAQ,IAAI,iBAAiB,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,QAAQ,CAAC,QAAQ,IAAI,KAAK,CAAC,aAAa,KAAK,QAAQ,CAAC,aAAa,IAAI,KAAK,CAAC,WAAW,KAAK,6CAA6C,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,iBAAiB,EAAE,QAAQ,EAAE,cAAc,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,WAAW,KAAK,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,iBAAiB,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,iBAAiB,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,UAAU,EAAE,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;QACvuB,OAAO,EAAE,iBAAiB,CAAC,KAAK,CAAC;KAClC,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/** Subsets of inputTokens, never added again to the existing token total. */
|
|
2
|
+
export interface CacheTokenUsage {
|
|
3
|
+
uncachedInputTokens?: number | null;
|
|
4
|
+
cacheReadInputTokens?: number | null;
|
|
5
|
+
cacheWriteInputTokens?: number | null;
|
|
6
|
+
}
|
|
7
|
+
export interface ProviderInvocation {
|
|
8
|
+
invocationId?: string;
|
|
9
|
+
ordinal?: number;
|
|
10
|
+
kind?: import('./role-routing.js').InvocationKind;
|
|
11
|
+
tier?: 'base' | 'escalation';
|
|
12
|
+
routeReason?: string;
|
|
13
|
+
contextMode?: 'full' | 'incremental';
|
|
14
|
+
promptBytes?: number;
|
|
15
|
+
contextBytes?: number;
|
|
16
|
+
handoffBytes?: number;
|
|
17
|
+
requestedEffort?: string | null;
|
|
18
|
+
provider: string;
|
|
19
|
+
/** Requested model; absent means the provider selected its default. */
|
|
20
|
+
model?: string;
|
|
21
|
+
status: 'succeeded' | 'failed';
|
|
22
|
+
/** Entire executor wall time, including its native tools and process startup. */
|
|
23
|
+
durationMs: number;
|
|
24
|
+
toolCalls: number;
|
|
25
|
+
usage: CacheTokenUsage & {
|
|
26
|
+
inputTokens: number | null;
|
|
27
|
+
outputTokens: number | null;
|
|
28
|
+
costUsd: number | null;
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
export declare const CACHE_TOKEN_KEYS: readonly ["uncachedInputTokens", "cacheReadInputTokens", "cacheWriteInputTokens"];
|
|
32
|
+
export declare function sumCacheUsage(values: CacheTokenUsage[]): CacheTokenUsage;
|
|
33
|
+
export interface EfficiencyTotals {
|
|
34
|
+
attempts: number;
|
|
35
|
+
measuredAttempts: number;
|
|
36
|
+
/** Completed phase wall time; null while an attempt has no end timestamp. */
|
|
37
|
+
durationMs: number | null;
|
|
38
|
+
agentDurationMs: number | null;
|
|
39
|
+
providerCalls: number | null;
|
|
40
|
+
toolCalls: number | null;
|
|
41
|
+
inputTokens: number | null;
|
|
42
|
+
outputTokens: number | null;
|
|
43
|
+
costUsd: number | null;
|
|
44
|
+
uncachedInputTokens: number | null;
|
|
45
|
+
cacheReadInputTokens: number | null;
|
|
46
|
+
cacheWriteInputTokens: number | null;
|
|
47
|
+
}
|
|
48
|
+
export interface RuntimeEfficiency {
|
|
49
|
+
schemaVersion: 1;
|
|
50
|
+
total: EfficiencyTotals;
|
|
51
|
+
phases: Array<EfficiencyTotals & {
|
|
52
|
+
stepId: string;
|
|
53
|
+
providers: string[];
|
|
54
|
+
models: string[];
|
|
55
|
+
}>;
|
|
56
|
+
}
|
|
57
|
+
export type PendingProviderInvocation = Pick<ProviderInvocation, 'provider' | 'model' | 'kind' | 'tier' | 'routeReason' | 'contextMode' | 'promptBytes' | 'contextBytes' | 'handoffBytes' | 'requestedEffort'> & {
|
|
58
|
+
invocationId: string;
|
|
59
|
+
ordinal: number;
|
|
60
|
+
};
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export const CACHE_TOKEN_KEYS = ['uncachedInputTokens', 'cacheReadInputTokens', 'cacheWriteInputTokens'];
|
|
2
|
+
export function sumCacheUsage(values) {
|
|
3
|
+
if (!values.some(value => CACHE_TOKEN_KEYS.some(key => value[key] !== undefined)))
|
|
4
|
+
return {};
|
|
5
|
+
return Object.fromEntries(CACHE_TOKEN_KEYS.map(key => [key, values.some(value => value[key] == null)
|
|
6
|
+
? null : values.reduce((sum, value) => sum + (value[key] ?? 0), 0)]));
|
|
7
|
+
}
|
|
8
|
+
//# sourceMappingURL=efficiency-types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"efficiency-types.js","sourceRoot":"","sources":["../../src/agent-runtime/efficiency-types.ts"],"names":[],"mappings":"AA0BA,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,qBAAqB,EAAE,sBAAsB,EAAE,uBAAuB,CAAU,CAAA;AAEjH,MAAM,UAAU,aAAa,CAAC,MAAyB;IACrD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,SAAS,CAAC,CAAC;QAAE,OAAO,EAAE,CAAA;IAC5F,OAAO,MAAM,CAAC,WAAW,CAAC,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;YAClG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;AACzE,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { WorkflowState } from './workflow-types.js';
|
|
2
|
+
import { type RuntimeEfficiency } from './efficiency-types.js';
|
|
3
|
+
export * from './efficiency-types.js';
|
|
4
|
+
/** Read-only projection: failed/superseded attempts remain spend, polling never changes it. */
|
|
5
|
+
export declare function runtimeEfficiency(state: WorkflowState): RuntimeEfficiency;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { CACHE_TOKEN_KEYS } from './efficiency-types.js';
|
|
2
|
+
export * from './efficiency-types.js';
|
|
3
|
+
function sum(values) {
|
|
4
|
+
return values.some(value => value == null || !Number.isFinite(value) || value < 0) ? null : values.reduce((total, value) => total + (value ?? 0), 0);
|
|
5
|
+
}
|
|
6
|
+
const AGENT_PHASES = new Set(['architect', 'developer', 'reviewer']);
|
|
7
|
+
function totals(attempts) {
|
|
8
|
+
const invocations = attempts.flatMap(attempt => attempt.invocations ?? []);
|
|
9
|
+
const measured = attempts.filter(attempt => !AGENT_PHASES.has(attempt.stepId) || attempt.invocations !== undefined);
|
|
10
|
+
const complete = measured.length === attempts.length && attempts.every(attempt => !attempt.pendingInvocations?.length);
|
|
11
|
+
const duration = (attempt) => attempt.completedAt ? Math.max(0, Date.parse(attempt.completedAt) - Date.parse(attempt.startedAt)) : null;
|
|
12
|
+
const cache = Object.fromEntries(CACHE_TOKEN_KEYS.map(key => [key, complete ? sum(invocations.map(call => call.usage[key])) : null]));
|
|
13
|
+
return {
|
|
14
|
+
attempts: attempts.length, measuredAttempts: measured.length,
|
|
15
|
+
durationMs: sum(attempts.map(duration)),
|
|
16
|
+
agentDurationMs: complete ? sum(invocations.map(call => call.durationMs)) : null,
|
|
17
|
+
providerCalls: complete ? invocations.length : null,
|
|
18
|
+
toolCalls: complete ? sum(invocations.map(call => call.toolCalls)) : null,
|
|
19
|
+
inputTokens: sum(attempts.map(attempt => attempt.usage?.inputTokens)),
|
|
20
|
+
outputTokens: sum(attempts.map(attempt => attempt.usage?.outputTokens)),
|
|
21
|
+
costUsd: sum(attempts.map(attempt => attempt.usage?.costUsd)),
|
|
22
|
+
...cache,
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
/** Read-only projection: failed/superseded attempts remain spend, polling never changes it. */
|
|
26
|
+
export function runtimeEfficiency(state) {
|
|
27
|
+
const phases = [...new Set(state.history.map(attempt => attempt.stepId))].map(stepId => {
|
|
28
|
+
const attempts = state.history.filter(attempt => attempt.stepId === stepId);
|
|
29
|
+
const calls = attempts.flatMap(attempt => attempt.invocations ?? []);
|
|
30
|
+
return { stepId, ...totals(attempts), providers: [...new Set(calls.map(call => call.provider))], models: [...new Set(calls.map(call => call.model).filter((model) => model !== undefined))] };
|
|
31
|
+
});
|
|
32
|
+
const pending = state.history.some(attempt => attempt.pendingInvocations?.length);
|
|
33
|
+
return { schemaVersion: 1, total: { ...totals(state.history), durationMs: state.usage.durationMs, inputTokens: pending ? null : state.usage.inputTokens, outputTokens: pending ? null : state.usage.outputTokens, costUsd: pending ? null : state.usage.costUsd }, phases };
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=efficiency.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"efficiency.js","sourceRoot":"","sources":["../../src/agent-runtime/efficiency.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAiD,MAAM,uBAAuB,CAAA;AACvG,cAAc,uBAAuB,CAAA;AAErC,SAAS,GAAG,CAAC,MAAwC;IACnD,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAS,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;AAC9J,CAAC;AACD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC,CAAA;AAEpE,SAAS,MAAM,CAAC,QAA6B;IAC3C,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC,CAAA;IAC1E,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,CAAC,CAAA;IACnH,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAA;IACtH,MAAM,QAAQ,GAAG,CAAC,OAA0B,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAC1J,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAA4D,CAAA;IAChM,OAAO;QACL,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,gBAAgB,EAAE,QAAQ,CAAC,MAAM;QAC5D,UAAU,EAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACvC,eAAe,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;QAChF,aAAa,EAAE,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;QACnD,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;QACzE,WAAW,EAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QACrE,YAAY,EAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;QACvE,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC7D,GAAG,KAAK;KACT,CAAA;AACH,CAAC;AAED,+FAA+F;AAC/F,MAAM,UAAU,iBAAiB,CAAC,KAAoB;IACpD,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;QACrF,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,CAAA;QAC3E,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC,CAAA;QACpE,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAA;IAChN,CAAC,CAAC,CAAA;IACF,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAA;IACjF,OAAO,EAAE,aAAa,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,CAAA;AAC7Q,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/** Frozen fixture definitions and independent assertions; no model call or install. */
|
|
2
|
+
export interface EvaluationCase {
|
|
3
|
+
id: string;
|
|
4
|
+
description: string;
|
|
5
|
+
repositories: string[];
|
|
6
|
+
source: string;
|
|
7
|
+
solution: string;
|
|
8
|
+
defects: string[];
|
|
9
|
+
oracle: string;
|
|
10
|
+
verification?: string;
|
|
11
|
+
correction?: 'verification' | 'review';
|
|
12
|
+
}
|
|
13
|
+
export declare const EVALUATION_CORPUS: EvaluationCase[];
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export const EVALUATION_CORPUS = [
|
|
2
|
+
{ id: 'static-tetris', verification: 'assert.deepStrictEqual(api.clearRows([[1,1]]), [[0,0]])', description: 'Clear complete rows in a static browser game without removing partial rows.', repositories: ['game'], source: 'exports.clearRows = board => board', solution: 'exports.clearRows = board => { const remaining = board.filter(row => row.some(cell => !cell)); return [...Array.from({length: board.length - remaining.length}, () => Array(board[0].length).fill(0)), ...remaining.map(row => [...row])]; }', defects: ['exports.clearRows = board => board', 'exports.clearRows = board => board.filter(row => row.every(Boolean))'], oracle: 'assert.deepStrictEqual(api.clearRows([[1,1],[1,0],[1,1]]), [[0,0],[0,0],[1,0]]); assert.deepStrictEqual(api.clearRows([[0,1],[1,0]]), [[0,1],[1,0]]);' },
|
|
3
|
+
{ id: 'local-tested-feature', verification: 'assert.strictEqual(api.subtotal([]), 0)', description: 'Calculate an integer-cent subtotal including repeated quantities and empty baskets.', repositories: ['shop'], source: 'exports.subtotal = items => 0', solution: 'exports.subtotal = items => items.reduce((sum, item) => sum + item.cents * item.quantity, 0)', defects: ['exports.subtotal = items => items.length', 'exports.subtotal = items => items.reduce((sum, item) => sum + item.cents, 0)'], oracle: 'assert.strictEqual(api.subtotal([]), 0); assert.strictEqual(api.subtotal([{cents: 125, quantity: 3}, {cents: 20, quantity: 2}]), 415);' },
|
|
4
|
+
{ id: 'cross-repository-contract', verification: 'assert.strictEqual(api.alert().active, true)', description: 'Keep both repositories on the same alert contract: an active severity and its informational flag.', repositories: ['front', 'back'], source: 'exports.alert = () => ({active: false})', solution: 'exports.alert = () => ({active: true, severity: "warning", informational: true})', defects: ['exports.alert = () => ({active: true, severity: "warning"})', 'exports.alert = () => ({active: true, severity: "error", informational: false})'], oracle: 'assert.deepStrictEqual(api.alert(), {active: true, severity: "warning", informational: true});' },
|
|
5
|
+
{ id: 'verification-correction', verification: 'assert.strictEqual(api.clamp(11, 0, 10), 10)', description: 'Clamp values to both boundaries, including an already valid value.', repositories: ['math'], source: 'exports.clamp = value => value', solution: 'exports.clamp = (value, min, max) => Math.min(max, Math.max(min, value))', defects: ['exports.clamp = (value, min, max) => Math.max(min, value)', 'exports.clamp = (value, min, max) => Math.min(max, value)'], oracle: 'assert.strictEqual(api.clamp(-1, 0, 10), 0); assert.strictEqual(api.clamp(11, 0, 10), 10); assert.strictEqual(api.clamp(5, 0, 10), 5);', correction: 'verification' },
|
|
6
|
+
{ id: 'review-correction', verification: 'assert.strictEqual(typeof api.unique, "function")', description: 'Deduplicate stable values while preserving the first occurrence order.', repositories: ['collections'], source: 'exports.unique = values => values', solution: 'exports.unique = values => [...new Set(values)]', defects: ['exports.unique = values => [...new Set(values)].sort()', 'exports.unique = values => values'], oracle: 'assert.deepStrictEqual(api.unique([3, 1, 3, 2, 1]), [3, 1, 2]); assert.deepStrictEqual(api.unique([]), []);', correction: 'review' },
|
|
7
|
+
];
|
|
8
|
+
//# sourceMappingURL=evaluation-corpus.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"evaluation-corpus.js","sourceRoot":"","sources":["../../src/agent-runtime/evaluation-corpus.ts"],"names":[],"mappings":"AAYA,MAAM,CAAC,MAAM,iBAAiB,GAAqB;IACjD,EAAE,EAAE,EAAE,eAAe,EAAE,YAAY,EAAE,yDAAyD,EAAE,WAAW,EAAE,6EAA6E,EAAE,YAAY,EAAE,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,oCAAoC,EAAE,QAAQ,EAAE,8OAA8O,EAAE,OAAO,EAAE,CAAC,oCAAoC,EAAE,sEAAsE,CAAC,EAAE,MAAM,EAAE,uJAAuJ,EAAE;IACtxB,EAAE,EAAE,EAAE,sBAAsB,EAAE,YAAY,EAAE,yCAAyC,EAAE,WAAW,EAAE,qFAAqF,EAAE,YAAY,EAAE,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,+BAA+B,EAAE,QAAQ,EAAE,8FAA8F,EAAE,OAAO,EAAE,CAAC,0CAA0C,EAAE,8EAA8E,CAAC,EAAE,MAAM,EAAE,wIAAwI,EAAE;IAC/nB,EAAE,EAAE,EAAE,2BAA2B,EAAE,YAAY,EAAE,8CAA8C,EAAE,WAAW,EAAE,mGAAmG,EAAE,YAAY,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,yCAAyC,EAAE,QAAQ,EAAE,kFAAkF,EAAE,OAAO,EAAE,CAAC,6DAA6D,EAAE,iFAAiF,CAAC,EAAE,MAAM,EAAE,gGAAgG,EAAE;IAC5oB,EAAE,EAAE,EAAE,yBAAyB,EAAE,YAAY,EAAE,8CAA8C,EAAE,WAAW,EAAE,oEAAoE,EAAE,YAAY,EAAE,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,gCAAgC,EAAE,QAAQ,EAAE,0EAA0E,EAAE,OAAO,EAAE,CAAC,2DAA2D,EAAE,2DAA2D,CAAC,EAAE,MAAM,EAAE,wIAAwI,EAAE,UAAU,EAAE,cAAc,EAAE;IAC7nB,EAAE,EAAE,EAAE,mBAAmB,EAAE,YAAY,EAAE,mDAAmD,EAAE,WAAW,EAAE,wEAAwE,EAAE,YAAY,EAAE,CAAC,aAAa,CAAC,EAAE,MAAM,EAAE,mCAAmC,EAAE,QAAQ,EAAE,iDAAiD,EAAE,OAAO,EAAE,CAAC,wDAAwD,EAAE,mCAAmC,CAAC,EAAE,MAAM,EAAE,6GAA6G,EAAE,UAAU,EAAE,QAAQ,EAAE;CACtjB,CAAA"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { RuntimeConfig } from './executor-types.js';
|
|
2
|
+
export interface EvaluationOptions {
|
|
3
|
+
output: string;
|
|
4
|
+
config?: RuntimeConfig;
|
|
5
|
+
maxCostUsd?: number;
|
|
6
|
+
real?: boolean;
|
|
7
|
+
repetitions?: number;
|
|
8
|
+
}
|
|
9
|
+
export declare function runEvaluation(options: EvaluationOptions): Promise<{
|
|
10
|
+
schemaVersion: number;
|
|
11
|
+
experiment: string;
|
|
12
|
+
noExtraInvocations: boolean;
|
|
13
|
+
mode: string;
|
|
14
|
+
runtimeIdentity: import("./runtime-identity.js").RuntimeIdentity;
|
|
15
|
+
corpusHash: string;
|
|
16
|
+
measurement: string;
|
|
17
|
+
monetaryTarget: number;
|
|
18
|
+
monetaryConclusion: string;
|
|
19
|
+
groups: {
|
|
20
|
+
[k: string]: {
|
|
21
|
+
samples: number;
|
|
22
|
+
independentlyAccepted: number;
|
|
23
|
+
costPerAcceptedUsd: number | null;
|
|
24
|
+
medianActiveDurationMs: number | null;
|
|
25
|
+
minActiveDurationMs: number | null;
|
|
26
|
+
maxActiveDurationMs: number | null;
|
|
27
|
+
standardDeviationMs: number | null;
|
|
28
|
+
};
|
|
29
|
+
};
|
|
30
|
+
noObservedQualityDrop: boolean;
|
|
31
|
+
promptPairs: {
|
|
32
|
+
caseId: unknown;
|
|
33
|
+
repeat: unknown;
|
|
34
|
+
fullBytes: number | null;
|
|
35
|
+
optimizedBytes: number | null;
|
|
36
|
+
reduction: number | null;
|
|
37
|
+
}[];
|
|
38
|
+
correctionPromptTargetMet: boolean;
|
|
39
|
+
sampleLimitations: string;
|
|
40
|
+
stopReason: string | null;
|
|
41
|
+
reportedSpendUsd: number;
|
|
42
|
+
observations: Record<string, unknown>[];
|
|
43
|
+
}>;
|