specrails-core 5.3.0 → 5.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/agent-runtime/capabilities.d.ts +28 -0
- package/dist/agent-runtime/capabilities.js +81 -0
- package/dist/agent-runtime/capabilities.js.map +1 -0
- package/dist/agent-runtime/cli-executor.d.ts +9 -0
- package/dist/agent-runtime/cli-executor.js +109 -34
- package/dist/agent-runtime/cli-executor.js.map +1 -1
- package/dist/agent-runtime/cli-process.d.ts +1 -0
- package/dist/agent-runtime/cli-process.js +13 -2
- package/dist/agent-runtime/cli-process.js.map +1 -1
- package/dist/agent-runtime/cli.d.ts +1 -0
- package/dist/agent-runtime/cli.js +60 -11
- package/dist/agent-runtime/cli.js.map +1 -1
- package/dist/agent-runtime/codex-schema.d.ts +4 -0
- package/dist/agent-runtime/codex-schema.js +31 -0
- package/dist/agent-runtime/codex-schema.js.map +1 -0
- package/dist/agent-runtime/compact/architect.d.ts +349 -0
- package/dist/agent-runtime/compact/architect.js +371 -0
- package/dist/agent-runtime/compact/architect.js.map +1 -0
- package/dist/agent-runtime/compact/chat-client.d.ts +83 -0
- package/dist/agent-runtime/compact/chat-client.js +307 -0
- package/dist/agent-runtime/compact/chat-client.js.map +1 -0
- package/dist/agent-runtime/compact/developer.d.ts +160 -0
- package/dist/agent-runtime/compact/developer.js +823 -0
- package/dist/agent-runtime/compact/developer.js.map +1 -0
- package/dist/agent-runtime/compact/environment.d.ts +62 -0
- package/dist/agent-runtime/compact/environment.js +213 -0
- package/dist/agent-runtime/compact/environment.js.map +1 -0
- package/dist/agent-runtime/compact/exit-code-honesty.d.ts +8 -0
- package/dist/agent-runtime/compact/exit-code-honesty.js +28 -0
- package/dist/agent-runtime/compact/exit-code-honesty.js.map +1 -0
- package/dist/agent-runtime/compact/guarded-loop.d.ts +73 -0
- package/dist/agent-runtime/compact/guarded-loop.js +320 -0
- package/dist/agent-runtime/compact/guarded-loop.js.map +1 -0
- package/dist/agent-runtime/compact/prompt-inputs.d.ts +42 -0
- package/dist/agent-runtime/compact/prompt-inputs.js +59 -0
- package/dist/agent-runtime/compact/prompt-inputs.js.map +1 -0
- package/dist/agent-runtime/compact/reviewer.d.ts +24 -0
- package/dist/agent-runtime/compact/reviewer.js +178 -0
- package/dist/agent-runtime/compact/reviewer.js.map +1 -0
- package/dist/agent-runtime/compact/step.d.ts +70 -0
- package/dist/agent-runtime/compact/step.js +136 -0
- package/dist/agent-runtime/compact/step.js.map +1 -0
- package/dist/agent-runtime/compact/test-reachability.d.ts +30 -0
- package/dist/agent-runtime/compact/test-reachability.js +147 -0
- package/dist/agent-runtime/compact/test-reachability.js.map +1 -0
- package/dist/agent-runtime/config.d.ts +4 -0
- package/dist/agent-runtime/config.js +146 -15
- package/dist/agent-runtime/config.js.map +1 -1
- package/dist/agent-runtime/core-host.d.ts +12 -4
- package/dist/agent-runtime/core-host.js +61 -21
- package/dist/agent-runtime/core-host.js.map +1 -1
- package/dist/agent-runtime/efficiency-summary.d.ts +67 -0
- package/dist/agent-runtime/efficiency-summary.js +52 -0
- package/dist/agent-runtime/efficiency-summary.js.map +1 -0
- package/dist/agent-runtime/efficiency-types.d.ts +60 -0
- package/dist/agent-runtime/efficiency-types.js +8 -0
- package/dist/agent-runtime/efficiency-types.js.map +1 -0
- package/dist/agent-runtime/efficiency.d.ts +5 -0
- package/dist/agent-runtime/efficiency.js +35 -0
- package/dist/agent-runtime/efficiency.js.map +1 -0
- package/dist/agent-runtime/evaluation-corpus.d.ts +13 -0
- package/dist/agent-runtime/evaluation-corpus.js +8 -0
- package/dist/agent-runtime/evaluation-corpus.js.map +1 -0
- package/dist/agent-runtime/evaluation.d.ts +43 -0
- package/dist/agent-runtime/evaluation.js +171 -0
- package/dist/agent-runtime/evaluation.js.map +1 -0
- package/dist/agent-runtime/executor-types.d.ts +71 -4
- package/dist/agent-runtime/executor-types.js +7 -2
- package/dist/agent-runtime/executor-types.js.map +1 -1
- package/dist/agent-runtime/executors.d.ts +1 -0
- package/dist/agent-runtime/executors.js +4 -0
- package/dist/agent-runtime/executors.js.map +1 -1
- package/dist/agent-runtime/graph/artifacts.d.ts +8 -22
- package/dist/agent-runtime/graph/artifacts.js +99 -73
- package/dist/agent-runtime/graph/artifacts.js.map +1 -1
- package/dist/agent-runtime/graph/nodes.d.ts +7 -2
- package/dist/agent-runtime/graph/nodes.js +207 -53
- package/dist/agent-runtime/graph/nodes.js.map +1 -1
- package/dist/agent-runtime/graph/roles.d.ts +11 -3
- package/dist/agent-runtime/graph/roles.js +109 -37
- package/dist/agent-runtime/graph/roles.js.map +1 -1
- package/dist/agent-runtime/graph/state.d.ts +10 -1
- package/dist/agent-runtime/graph/state.js +2 -1
- package/dist/agent-runtime/graph/state.js.map +1 -1
- package/dist/agent-runtime/guardrails.d.ts +15 -0
- package/dist/agent-runtime/guardrails.js +76 -0
- package/dist/agent-runtime/guardrails.js.map +1 -0
- package/dist/agent-runtime/index.d.ts +7 -1
- package/dist/agent-runtime/index.js +7 -1
- package/dist/agent-runtime/index.js.map +1 -1
- package/dist/agent-runtime/kimi-acp.d.ts +4 -0
- package/dist/agent-runtime/kimi-acp.js +20 -9
- package/dist/agent-runtime/kimi-acp.js.map +1 -1
- package/dist/agent-runtime/openai-executor.d.ts +33 -1
- package/dist/agent-runtime/openai-executor.js +76 -113
- package/dist/agent-runtime/openai-executor.js.map +1 -1
- package/dist/agent-runtime/openspec-tool-server.d.ts +1 -0
- package/dist/agent-runtime/openspec-tool-server.js +37 -0
- package/dist/agent-runtime/openspec-tool-server.js.map +1 -0
- package/dist/agent-runtime/openspec.d.ts +198 -0
- package/dist/agent-runtime/openspec.js +329 -0
- package/dist/agent-runtime/openspec.js.map +1 -0
- package/dist/agent-runtime/prompts.d.ts +24 -1
- package/dist/agent-runtime/prompts.js +112 -43
- package/dist/agent-runtime/prompts.js.map +1 -1
- package/dist/agent-runtime/provider-diagnostic.d.ts +2 -0
- package/dist/agent-runtime/provider-diagnostic.js +25 -0
- package/dist/agent-runtime/provider-diagnostic.js.map +1 -0
- package/dist/agent-runtime/repository-context.d.ts +25 -0
- package/dist/agent-runtime/repository-context.js +127 -0
- package/dist/agent-runtime/repository-context.js.map +1 -0
- package/dist/agent-runtime/review-context.d.ts +14 -0
- package/dist/agent-runtime/review-context.js +41 -0
- package/dist/agent-runtime/review-context.js.map +1 -0
- package/dist/agent-runtime/role-routing.d.ts +10 -0
- package/dist/agent-runtime/role-routing.js +30 -0
- package/dist/agent-runtime/role-routing.js.map +1 -0
- package/dist/agent-runtime/role-state.d.ts +19 -0
- package/dist/agent-runtime/role-state.js +24 -0
- package/dist/agent-runtime/role-state.js.map +1 -0
- package/dist/agent-runtime/runtime-identity.d.ts +11 -0
- package/dist/agent-runtime/runtime-identity.js +33 -0
- package/dist/agent-runtime/runtime-identity.js.map +1 -0
- package/dist/agent-runtime/tool-event.d.ts +3 -0
- package/dist/agent-runtime/tool-event.js +24 -0
- package/dist/agent-runtime/tool-event.js.map +1 -0
- package/dist/agent-runtime/verification-plan.d.ts +55 -0
- package/dist/agent-runtime/verification-plan.js +206 -0
- package/dist/agent-runtime/verification-plan.js.map +1 -0
- package/dist/agent-runtime/workflow-types.d.ts +19 -1
- package/dist/agent-runtime/workflow.js +29 -2
- package/dist/agent-runtime/workflow.js.map +1 -1
- package/dist/agent-runtime/workspace-tools.d.ts +4 -0
- package/dist/agent-runtime/workspace-tools.js +182 -20
- package/dist/agent-runtime/workspace-tools.js.map +1 -1
- package/dist/installer/phases/scaffold.js +39 -109
- package/dist/installer/phases/scaffold.js.map +1 -1
- package/dist/installer/runtime/pipeline-state.d.ts +143 -1
- package/dist/installer/runtime/pipeline-state.js +510 -30
- package/dist/installer/runtime/pipeline-state.js.map +1 -1
- package/docs/agent-runtime-efficiency.md +65 -0
- package/docs/agent-runtime.md +86 -15
- package/integration-contract.json +2 -1
- package/package.json +5 -2
- package/schemas/agent-runtime.schema.json +34 -4
- package/schemas/fixtures/runtime-efficiency-summary.v1.json +790 -0
- package/templates/codex-skills/batch-implement/SKILL.md +33 -58
- package/templates/codex-skills/implement/SKILL.md +21 -124
- package/templates/codex-skills/retry/SKILL.md +8 -34
- package/templates/commands/specrails/batch-implement.md +21 -16
- package/templates/commands/specrails/implement.md +17 -276
- package/templates/commands/specrails/retry.md +6 -34
- package/templates/gemini-commands/batch-implement.toml +34 -28
- package/templates/gemini-commands/implement.toml +34 -55
- package/templates/gemini-commands/retry.toml +10 -16
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { readVerificationEvidence } from '../installer/runtime/pipeline-state.js';
|
|
2
|
+
import { runtimeEfficiency } from './efficiency.js';
|
|
3
|
+
/** Compact derived view. Durable invocation/evidence IDs remain the accounting authority. */
|
|
4
|
+
export function efficiencySummary(state, context, config, completion, identity) {
|
|
5
|
+
const invocations = state.history.flatMap(attempt => [...(attempt.invocations ?? []), ...(attempt.pendingInvocations ?? [])].map(call => ({ ...call, role: attempt.stepId, attemptId: attempt.id })));
|
|
6
|
+
const kinds = ['initial', 'correction', 'repair', 'deepen', 'session-fallback'];
|
|
7
|
+
const byKind = Object.fromEntries(kinds.map(kind => [kind, invocations.filter(call => call.kind === kind).length]));
|
|
8
|
+
const measured = state.history.every(attempt => !attempt.pendingInvocations?.length) && state.history.every(attempt => !['architect', 'developer', 'reviewer'].includes(attempt.stepId) || attempt.invocations !== undefined) && invocations.every(call => call.kind !== undefined && call.promptBytes !== undefined);
|
|
9
|
+
const bytes = (key) => invocations.every(call => call[key] !== undefined) ? invocations.reduce((sum, call) => sum + call[key], 0) : null;
|
|
10
|
+
const checkIds = new Set();
|
|
11
|
+
let checks = [], evidenceAvailable = true;
|
|
12
|
+
try {
|
|
13
|
+
let cursor;
|
|
14
|
+
do {
|
|
15
|
+
const page = readVerificationEvidence(context, { limit: 100, ...(cursor ? { cursor } : {}) });
|
|
16
|
+
if (!page.available) {
|
|
17
|
+
evidenceAvailable = false;
|
|
18
|
+
break;
|
|
19
|
+
}
|
|
20
|
+
for (const row of page.items ?? [])
|
|
21
|
+
if (typeof row.id === 'string' && !checkIds.has(row.id)) {
|
|
22
|
+
checkIds.add(row.id);
|
|
23
|
+
checks.push(row);
|
|
24
|
+
}
|
|
25
|
+
if (checkIds.size > 10000)
|
|
26
|
+
throw new Error('Evidence summary limit');
|
|
27
|
+
cursor = 'nextCursor' in page ? page.nextCursor : undefined;
|
|
28
|
+
} while (cursor);
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
checks = [];
|
|
32
|
+
evidenceAvailable = false;
|
|
33
|
+
}
|
|
34
|
+
const checksComplete = evidenceAvailable && checks.every(check => check.status !== 'interrupted' && typeof check.durationMs === 'number');
|
|
35
|
+
const escalatedRoles = new Set();
|
|
36
|
+
const escalations = invocations.filter(call => { if (call.tier !== 'escalation' || escalatedRoles.has(call.role))
|
|
37
|
+
return false; escalatedRoles.add(call.role); return true; }).map(call => ({ role: call.role, attemptId: call.attemptId, provider: call.provider, model: call.model ?? null, effort: call.requestedEffort ?? null, reason: call.routeReason ?? null }));
|
|
38
|
+
return {
|
|
39
|
+
schemaVersion: 1, currentEvidenceIds: [...(identity?.verification?.receipt?.commands.flatMap(command => command.evidenceId ? [command.evidenceId] : []) ?? []), ...(identity?.verification?.receipt?.notRunEvidenceIds ?? [])].slice(0, 100), planHash: identity?.planHash ?? null, candidateHash: identity?.candidateHash ?? null, runId: state.runId, workflowVersion: state.workflowVersion,
|
|
40
|
+
technicalAcceptance: completion?.validation === 'verified' ? 'validated' : completion?.validation === 'with-exceptions' ? 'with-exceptions' : completion?.validation === 'blocked' ? 'blocked' : 'pending',
|
|
41
|
+
archive: completion?.archive ?? 'pending', delivery: completion?.delivery ?? 'pending',
|
|
42
|
+
roles: ['architect', 'developer', 'reviewer'].map(role => {
|
|
43
|
+
const selected = config?.agents[role], latest = invocations.filter(call => call.role === role).at(-1);
|
|
44
|
+
return { role, provider: latest?.provider ?? selected?.provider ?? null, model: latest?.model ?? selected?.model ?? null, effort: latest ? latest.requestedEffort ?? null : selected?.effort ?? null, observedModel: null, observedEffort: null, origin: 'frozen-runtime-config', tier: latest?.tier ?? 'base' };
|
|
45
|
+
}),
|
|
46
|
+
invocations: { byKind, total: invocations.length, complete: measured, promptBytes: bytes('promptBytes'), contextBytes: bytes('contextBytes'), handoffBytes: bytes('handoffBytes'), fullContexts: invocations.filter(call => call.contextMode === 'full').length, incrementalContexts: invocations.filter(call => call.contextMode === 'incremental').length },
|
|
47
|
+
escalations: escalations.slice(-16), escalationsTruncated: escalations.length > 16,
|
|
48
|
+
checks: { invalidated: identity && evidenceAvailable ? checks.filter(check => check.planHash !== identity.planHash || check.candidateHash !== identity.candidateHash || check.reuseReason === 'snapshot-inputs-changed-during-verification').length : null, available: evidenceAvailable, complete: checksComplete, executed: checksComplete ? checks.filter(check => check.disposition === 'executed').length : null, reused: evidenceAvailable ? checks.filter(check => check.disposition === 'reused').length : null, notRun: evidenceAvailable ? checks.filter(check => check.disposition === 'not-run').length : null, durationMs: checksComplete ? checks.reduce((sum, check) => sum + (typeof check.durationMs === 'number' ? check.durationMs : 0), 0) : null },
|
|
49
|
+
metrics: runtimeEfficiency(state),
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=efficiency-summary.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"efficiency-summary.js","sourceRoot":"","sources":["../../src/agent-runtime/efficiency-summary.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,EAAiD,MAAM,wCAAwC,CAAA;AAGhI,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAEnD,6FAA6F;AAC7F,MAAM,UAAU,iBAAiB,CAAC,KAAoB,EAAE,OAAwB,EAAE,MAAiC,EAAE,UAA+B,EAAE,QAAgK;IACpT,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAA;IACrM,MAAM,KAAK,GAAG,CAAC,SAAS,EAAE,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,kBAAkB,CAAU,CAAA;IACxF,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;IACnH,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,CAAC,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,CAAA;IACrT,MAAM,KAAK,GAAG,CAAC,GAAoD,EAAE,EAAE,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAC1L,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA;IAClC,IAAI,MAAM,GAAmC,EAAE,EAAE,iBAAiB,GAAG,IAAI,CAAA;IACzE,IAAI,CAAC;QACH,IAAI,MAA0B,CAAA;QAC9B,GAAG,CAAC;YACF,MAAM,IAAI,GAAG,wBAAwB,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAA;YAC7F,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;gBAAC,iBAAiB,GAAG,KAAK,CAAC;gBAAC,MAAK;YAAC,CAAC;YACzD,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,IAAI,EAAE;gBAAE,IAAI,OAAO,GAAG,CAAC,EAAE,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;oBAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBAAC,CAAC;YACvI,IAAI,QAAQ,CAAC,IAAI,GAAG,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAA;YACpE,MAAM,GAAG,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAA;QAC7D,CAAC,QAAQ,MAAM,EAAC;IAClB,CAAC;IAAC,MAAM,CAAC;QAAC,MAAM,GAAG,EAAE,CAAC;QAAC,iBAAiB,GAAG,KAAK,CAAA;IAAC,CAAC;IAClD,MAAM,cAAc,GAAG,iBAAiB,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,aAAa,IAAI,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAA;IACzI,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAA;IACxC,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,IAAI,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,IAAI,CAAA,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,eAAe,IAAI,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,IAAI,IAAI,EAAE,CAAC,CAAC,CAAA;IACvW,OAAO;QACL,aAAa,EAAE,CAAU,EAAE,kBAAkB,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,iBAAiB,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,IAAI,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,aAAa,IAAI,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,eAAe,EAAE,KAAK,CAAC,eAAe;QACvY,mBAAmB,EAAE,UAAU,EAAE,UAAU,KAAK,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,EAAE,UAAU,KAAK,iBAAiB,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,UAAU,EAAE,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;QAC1M,OAAO,EAAE,UAAU,EAAE,OAAO,IAAI,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,IAAI,SAAS;QACtF,KAAK,EAAG,CAAC,WAAW,EAAE,WAAW,EAAE,UAAU,CAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;YAClE,MAAM,QAAQ,GAAG,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;YACrG,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,IAAI,QAAQ,EAAE,QAAQ,IAAI,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,IAAI,QAAQ,EAAE,KAAK,IAAI,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,IAAI,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,MAAM,IAAI,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,EAAE,uBAAuB,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,IAAI,MAAM,EAAE,CAAA;QAClT,CAAC,CAAC;QACF,WAAW,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,aAAa,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,cAAc,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,cAAc,CAAC,EAAE,YAAY,EAAE,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,KAAK,MAAM,CAAC,CAAC,MAAM,EAAE,mBAAmB,EAAE,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,KAAK,aAAa,CAAC,CAAC,MAAM,EAAE;QAC7V,WAAW,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,oBAAoB,EAAE,WAAW,CAAC,MAAM,GAAG,EAAE;QAClF,MAAM,EAAE,EAAE,WAAW,EAAE,QAAQ,IAAI,iBAAiB,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,QAAQ,CAAC,QAAQ,IAAI,KAAK,CAAC,aAAa,KAAK,QAAQ,CAAC,aAAa,IAAI,KAAK,CAAC,WAAW,KAAK,6CAA6C,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,iBAAiB,EAAE,QAAQ,EAAE,cAAc,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,WAAW,KAAK,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,iBAAiB,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,iBAAiB,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,UAAU,EAAE,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;QACvuB,OAAO,EAAE,iBAAiB,CAAC,KAAK,CAAC;KAClC,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/** Subsets of inputTokens, never added again to the existing token total. */
|
|
2
|
+
export interface CacheTokenUsage {
|
|
3
|
+
uncachedInputTokens?: number | null;
|
|
4
|
+
cacheReadInputTokens?: number | null;
|
|
5
|
+
cacheWriteInputTokens?: number | null;
|
|
6
|
+
}
|
|
7
|
+
export interface ProviderInvocation {
|
|
8
|
+
invocationId?: string;
|
|
9
|
+
ordinal?: number;
|
|
10
|
+
kind?: import('./role-routing.js').InvocationKind;
|
|
11
|
+
tier?: 'base' | 'escalation';
|
|
12
|
+
routeReason?: string;
|
|
13
|
+
contextMode?: 'full' | 'incremental';
|
|
14
|
+
promptBytes?: number;
|
|
15
|
+
contextBytes?: number;
|
|
16
|
+
handoffBytes?: number;
|
|
17
|
+
requestedEffort?: string | null;
|
|
18
|
+
provider: string;
|
|
19
|
+
/** Requested model; absent means the provider selected its default. */
|
|
20
|
+
model?: string;
|
|
21
|
+
status: 'succeeded' | 'failed';
|
|
22
|
+
/** Entire executor wall time, including its native tools and process startup. */
|
|
23
|
+
durationMs: number;
|
|
24
|
+
toolCalls: number;
|
|
25
|
+
usage: CacheTokenUsage & {
|
|
26
|
+
inputTokens: number | null;
|
|
27
|
+
outputTokens: number | null;
|
|
28
|
+
costUsd: number | null;
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
export declare const CACHE_TOKEN_KEYS: readonly ["uncachedInputTokens", "cacheReadInputTokens", "cacheWriteInputTokens"];
|
|
32
|
+
export declare function sumCacheUsage(values: CacheTokenUsage[]): CacheTokenUsage;
|
|
33
|
+
export interface EfficiencyTotals {
|
|
34
|
+
attempts: number;
|
|
35
|
+
measuredAttempts: number;
|
|
36
|
+
/** Completed phase wall time; null while an attempt has no end timestamp. */
|
|
37
|
+
durationMs: number | null;
|
|
38
|
+
agentDurationMs: number | null;
|
|
39
|
+
providerCalls: number | null;
|
|
40
|
+
toolCalls: number | null;
|
|
41
|
+
inputTokens: number | null;
|
|
42
|
+
outputTokens: number | null;
|
|
43
|
+
costUsd: number | null;
|
|
44
|
+
uncachedInputTokens: number | null;
|
|
45
|
+
cacheReadInputTokens: number | null;
|
|
46
|
+
cacheWriteInputTokens: number | null;
|
|
47
|
+
}
|
|
48
|
+
export interface RuntimeEfficiency {
|
|
49
|
+
schemaVersion: 1;
|
|
50
|
+
total: EfficiencyTotals;
|
|
51
|
+
phases: Array<EfficiencyTotals & {
|
|
52
|
+
stepId: string;
|
|
53
|
+
providers: string[];
|
|
54
|
+
models: string[];
|
|
55
|
+
}>;
|
|
56
|
+
}
|
|
57
|
+
export type PendingProviderInvocation = Pick<ProviderInvocation, 'provider' | 'model' | 'kind' | 'tier' | 'routeReason' | 'contextMode' | 'promptBytes' | 'contextBytes' | 'handoffBytes' | 'requestedEffort'> & {
|
|
58
|
+
invocationId: string;
|
|
59
|
+
ordinal: number;
|
|
60
|
+
};
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export const CACHE_TOKEN_KEYS = ['uncachedInputTokens', 'cacheReadInputTokens', 'cacheWriteInputTokens'];
|
|
2
|
+
export function sumCacheUsage(values) {
|
|
3
|
+
if (!values.some(value => CACHE_TOKEN_KEYS.some(key => value[key] !== undefined)))
|
|
4
|
+
return {};
|
|
5
|
+
return Object.fromEntries(CACHE_TOKEN_KEYS.map(key => [key, values.some(value => value[key] == null)
|
|
6
|
+
? null : values.reduce((sum, value) => sum + (value[key] ?? 0), 0)]));
|
|
7
|
+
}
|
|
8
|
+
//# sourceMappingURL=efficiency-types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"efficiency-types.js","sourceRoot":"","sources":["../../src/agent-runtime/efficiency-types.ts"],"names":[],"mappings":"AA0BA,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,qBAAqB,EAAE,sBAAsB,EAAE,uBAAuB,CAAU,CAAA;AAEjH,MAAM,UAAU,aAAa,CAAC,MAAyB;IACrD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,SAAS,CAAC,CAAC;QAAE,OAAO,EAAE,CAAA;IAC5F,OAAO,MAAM,CAAC,WAAW,CAAC,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;YAClG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;AACzE,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { WorkflowState } from './workflow-types.js';
|
|
2
|
+
import { type RuntimeEfficiency } from './efficiency-types.js';
|
|
3
|
+
export * from './efficiency-types.js';
|
|
4
|
+
/** Read-only projection: failed/superseded attempts remain spend, polling never changes it. */
|
|
5
|
+
export declare function runtimeEfficiency(state: WorkflowState): RuntimeEfficiency;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { CACHE_TOKEN_KEYS } from './efficiency-types.js';
|
|
2
|
+
export * from './efficiency-types.js';
|
|
3
|
+
function sum(values) {
|
|
4
|
+
return values.some(value => value == null || !Number.isFinite(value) || value < 0) ? null : values.reduce((total, value) => total + (value ?? 0), 0);
|
|
5
|
+
}
|
|
6
|
+
const AGENT_PHASES = new Set(['architect', 'developer', 'reviewer']);
|
|
7
|
+
function totals(attempts) {
|
|
8
|
+
const invocations = attempts.flatMap(attempt => attempt.invocations ?? []);
|
|
9
|
+
const measured = attempts.filter(attempt => !AGENT_PHASES.has(attempt.stepId) || attempt.invocations !== undefined);
|
|
10
|
+
const complete = measured.length === attempts.length && attempts.every(attempt => !attempt.pendingInvocations?.length);
|
|
11
|
+
const duration = (attempt) => attempt.completedAt ? Math.max(0, Date.parse(attempt.completedAt) - Date.parse(attempt.startedAt)) : null;
|
|
12
|
+
const cache = Object.fromEntries(CACHE_TOKEN_KEYS.map(key => [key, complete ? sum(invocations.map(call => call.usage[key])) : null]));
|
|
13
|
+
return {
|
|
14
|
+
attempts: attempts.length, measuredAttempts: measured.length,
|
|
15
|
+
durationMs: sum(attempts.map(duration)),
|
|
16
|
+
agentDurationMs: complete ? sum(invocations.map(call => call.durationMs)) : null,
|
|
17
|
+
providerCalls: complete ? invocations.length : null,
|
|
18
|
+
toolCalls: complete ? sum(invocations.map(call => call.toolCalls)) : null,
|
|
19
|
+
inputTokens: sum(attempts.map(attempt => attempt.usage?.inputTokens)),
|
|
20
|
+
outputTokens: sum(attempts.map(attempt => attempt.usage?.outputTokens)),
|
|
21
|
+
costUsd: sum(attempts.map(attempt => attempt.usage?.costUsd)),
|
|
22
|
+
...cache,
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
/** Read-only projection: failed/superseded attempts remain spend, polling never changes it. */
|
|
26
|
+
export function runtimeEfficiency(state) {
|
|
27
|
+
const phases = [...new Set(state.history.map(attempt => attempt.stepId))].map(stepId => {
|
|
28
|
+
const attempts = state.history.filter(attempt => attempt.stepId === stepId);
|
|
29
|
+
const calls = attempts.flatMap(attempt => attempt.invocations ?? []);
|
|
30
|
+
return { stepId, ...totals(attempts), providers: [...new Set(calls.map(call => call.provider))], models: [...new Set(calls.map(call => call.model).filter((model) => model !== undefined))] };
|
|
31
|
+
});
|
|
32
|
+
const pending = state.history.some(attempt => attempt.pendingInvocations?.length);
|
|
33
|
+
return { schemaVersion: 1, total: { ...totals(state.history), durationMs: state.usage.durationMs, inputTokens: pending ? null : state.usage.inputTokens, outputTokens: pending ? null : state.usage.outputTokens, costUsd: pending ? null : state.usage.costUsd }, phases };
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=efficiency.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"efficiency.js","sourceRoot":"","sources":["../../src/agent-runtime/efficiency.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAiD,MAAM,uBAAuB,CAAA;AACvG,cAAc,uBAAuB,CAAA;AAErC,SAAS,GAAG,CAAC,MAAwC;IACnD,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAS,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;AAC9J,CAAC;AACD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC,CAAA;AAEpE,SAAS,MAAM,CAAC,QAA6B;IAC3C,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC,CAAA;IAC1E,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,CAAC,CAAA;IACnH,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAA;IACtH,MAAM,QAAQ,GAAG,CAAC,OAA0B,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAC1J,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAA4D,CAAA;IAChM,OAAO;QACL,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,gBAAgB,EAAE,QAAQ,CAAC,MAAM;QAC5D,UAAU,EAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACvC,eAAe,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;QAChF,aAAa,EAAE,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;QACnD,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;QACzE,WAAW,EAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QACrE,YAAY,EAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;QACvE,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC7D,GAAG,KAAK;KACT,CAAA;AACH,CAAC;AAED,+FAA+F;AAC/F,MAAM,UAAU,iBAAiB,CAAC,KAAoB;IACpD,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;QACrF,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,CAAA;QAC3E,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC,CAAA;QACpE,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAA;IAChN,CAAC,CAAC,CAAA;IACF,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAA;IACjF,OAAO,EAAE,aAAa,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,CAAA;AAC7Q,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/** Frozen fixture definitions and independent assertions; no model call or install. */
|
|
2
|
+
export interface EvaluationCase {
|
|
3
|
+
id: string;
|
|
4
|
+
description: string;
|
|
5
|
+
repositories: string[];
|
|
6
|
+
source: string;
|
|
7
|
+
solution: string;
|
|
8
|
+
defects: string[];
|
|
9
|
+
oracle: string;
|
|
10
|
+
verification?: string;
|
|
11
|
+
correction?: 'verification' | 'review';
|
|
12
|
+
}
|
|
13
|
+
export declare const EVALUATION_CORPUS: EvaluationCase[];
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export const EVALUATION_CORPUS = [
|
|
2
|
+
{ id: 'static-tetris', verification: 'assert.deepStrictEqual(api.clearRows([[1,1]]), [[0,0]])', description: 'Clear complete rows in a static browser game without removing partial rows.', repositories: ['game'], source: 'exports.clearRows = board => board', solution: 'exports.clearRows = board => { const remaining = board.filter(row => row.some(cell => !cell)); return [...Array.from({length: board.length - remaining.length}, () => Array(board[0].length).fill(0)), ...remaining.map(row => [...row])]; }', defects: ['exports.clearRows = board => board', 'exports.clearRows = board => board.filter(row => row.every(Boolean))'], oracle: 'assert.deepStrictEqual(api.clearRows([[1,1],[1,0],[1,1]]), [[0,0],[0,0],[1,0]]); assert.deepStrictEqual(api.clearRows([[0,1],[1,0]]), [[0,1],[1,0]]);' },
|
|
3
|
+
{ id: 'local-tested-feature', verification: 'assert.strictEqual(api.subtotal([]), 0)', description: 'Calculate an integer-cent subtotal including repeated quantities and empty baskets.', repositories: ['shop'], source: 'exports.subtotal = items => 0', solution: 'exports.subtotal = items => items.reduce((sum, item) => sum + item.cents * item.quantity, 0)', defects: ['exports.subtotal = items => items.length', 'exports.subtotal = items => items.reduce((sum, item) => sum + item.cents, 0)'], oracle: 'assert.strictEqual(api.subtotal([]), 0); assert.strictEqual(api.subtotal([{cents: 125, quantity: 3}, {cents: 20, quantity: 2}]), 415);' },
|
|
4
|
+
{ id: 'cross-repository-contract', verification: 'assert.strictEqual(api.alert().active, true)', description: 'Keep both repositories on the same alert contract: an active severity and its informational flag.', repositories: ['front', 'back'], source: 'exports.alert = () => ({active: false})', solution: 'exports.alert = () => ({active: true, severity: "warning", informational: true})', defects: ['exports.alert = () => ({active: true, severity: "warning"})', 'exports.alert = () => ({active: true, severity: "error", informational: false})'], oracle: 'assert.deepStrictEqual(api.alert(), {active: true, severity: "warning", informational: true});' },
|
|
5
|
+
{ id: 'verification-correction', verification: 'assert.strictEqual(api.clamp(11, 0, 10), 10)', description: 'Clamp values to both boundaries, including an already valid value.', repositories: ['math'], source: 'exports.clamp = value => value', solution: 'exports.clamp = (value, min, max) => Math.min(max, Math.max(min, value))', defects: ['exports.clamp = (value, min, max) => Math.max(min, value)', 'exports.clamp = (value, min, max) => Math.min(max, value)'], oracle: 'assert.strictEqual(api.clamp(-1, 0, 10), 0); assert.strictEqual(api.clamp(11, 0, 10), 10); assert.strictEqual(api.clamp(5, 0, 10), 5);', correction: 'verification' },
|
|
6
|
+
{ id: 'review-correction', verification: 'assert.strictEqual(typeof api.unique, "function")', description: 'Deduplicate stable values while preserving the first occurrence order.', repositories: ['collections'], source: 'exports.unique = values => values', solution: 'exports.unique = values => [...new Set(values)]', defects: ['exports.unique = values => [...new Set(values)].sort()', 'exports.unique = values => values'], oracle: 'assert.deepStrictEqual(api.unique([3, 1, 3, 2, 1]), [3, 1, 2]); assert.deepStrictEqual(api.unique([]), []);', correction: 'review' },
|
|
7
|
+
];
|
|
8
|
+
//# sourceMappingURL=evaluation-corpus.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"evaluation-corpus.js","sourceRoot":"","sources":["../../src/agent-runtime/evaluation-corpus.ts"],"names":[],"mappings":"AAYA,MAAM,CAAC,MAAM,iBAAiB,GAAqB;IACjD,EAAE,EAAE,EAAE,eAAe,EAAE,YAAY,EAAE,yDAAyD,EAAE,WAAW,EAAE,6EAA6E,EAAE,YAAY,EAAE,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,oCAAoC,EAAE,QAAQ,EAAE,8OAA8O,EAAE,OAAO,EAAE,CAAC,oCAAoC,EAAE,sEAAsE,CAAC,EAAE,MAAM,EAAE,uJAAuJ,EAAE;IACtxB,EAAE,EAAE,EAAE,sBAAsB,EAAE,YAAY,EAAE,yCAAyC,EAAE,WAAW,EAAE,qFAAqF,EAAE,YAAY,EAAE,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,+BAA+B,EAAE,QAAQ,EAAE,8FAA8F,EAAE,OAAO,EAAE,CAAC,0CAA0C,EAAE,8EAA8E,CAAC,EAAE,MAAM,EAAE,wIAAwI,EAAE;IAC/nB,EAAE,EAAE,EAAE,2BAA2B,EAAE,YAAY,EAAE,8CAA8C,EAAE,WAAW,EAAE,mGAAmG,EAAE,YAAY,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,yCAAyC,EAAE,QAAQ,EAAE,kFAAkF,EAAE,OAAO,EAAE,CAAC,6DAA6D,EAAE,iFAAiF,CAAC,EAAE,MAAM,EAAE,gGAAgG,EAAE;IAC5oB,EAAE,EAAE,EAAE,yBAAyB,EAAE,YAAY,EAAE,8CAA8C,EAAE,WAAW,EAAE,oEAAoE,EAAE,YAAY,EAAE,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,gCAAgC,EAAE,QAAQ,EAAE,0EAA0E,EAAE,OAAO,EAAE,CAAC,2DAA2D,EAAE,2DAA2D,CAAC,EAAE,MAAM,EAAE,wIAAwI,EAAE,UAAU,EAAE,cAAc,EAAE;IAC7nB,EAAE,EAAE,EAAE,mBAAmB,EAAE,YAAY,EAAE,mDAAmD,EAAE,WAAW,EAAE,wEAAwE,EAAE,YAAY,EAAE,CAAC,aAAa,CAAC,EAAE,MAAM,EAAE,mCAAmC,EAAE,QAAQ,EAAE,iDAAiD,EAAE,OAAO,EAAE,CAAC,wDAAwD,EAAE,mCAAmC,CAAC,EAAE,MAAM,EAAE,6GAA6G,EAAE,UAAU,EAAE,QAAQ,EAAE;CACtjB,CAAA"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { RuntimeConfig } from './executor-types.js';
|
|
2
|
+
export interface EvaluationOptions {
|
|
3
|
+
output: string;
|
|
4
|
+
config?: RuntimeConfig;
|
|
5
|
+
maxCostUsd?: number;
|
|
6
|
+
real?: boolean;
|
|
7
|
+
repetitions?: number;
|
|
8
|
+
}
|
|
9
|
+
export declare function runEvaluation(options: EvaluationOptions): Promise<{
|
|
10
|
+
schemaVersion: number;
|
|
11
|
+
experiment: string;
|
|
12
|
+
noExtraInvocations: boolean;
|
|
13
|
+
mode: string;
|
|
14
|
+
runtimeIdentity: import("./runtime-identity.js").RuntimeIdentity;
|
|
15
|
+
corpusHash: string;
|
|
16
|
+
measurement: string;
|
|
17
|
+
monetaryTarget: number;
|
|
18
|
+
monetaryConclusion: string;
|
|
19
|
+
groups: {
|
|
20
|
+
[k: string]: {
|
|
21
|
+
samples: number;
|
|
22
|
+
independentlyAccepted: number;
|
|
23
|
+
costPerAcceptedUsd: number | null;
|
|
24
|
+
medianActiveDurationMs: number | null;
|
|
25
|
+
minActiveDurationMs: number | null;
|
|
26
|
+
maxActiveDurationMs: number | null;
|
|
27
|
+
standardDeviationMs: number | null;
|
|
28
|
+
};
|
|
29
|
+
};
|
|
30
|
+
noObservedQualityDrop: boolean;
|
|
31
|
+
promptPairs: {
|
|
32
|
+
caseId: unknown;
|
|
33
|
+
repeat: unknown;
|
|
34
|
+
fullBytes: number | null;
|
|
35
|
+
optimizedBytes: number | null;
|
|
36
|
+
reduction: number | null;
|
|
37
|
+
}[];
|
|
38
|
+
correctionPromptTargetMet: boolean;
|
|
39
|
+
sampleLimitations: string;
|
|
40
|
+
stopReason: string | null;
|
|
41
|
+
reportedSpendUsd: number;
|
|
42
|
+
observations: Record<string, unknown>[];
|
|
43
|
+
}>;
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import { execFileSync, spawnSync } from 'node:child_process';
|
|
2
|
+
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { tmpdir } from 'node:os';
|
|
5
|
+
import { randomUUID } from 'node:crypto';
|
|
6
|
+
import { EVALUATION_CORPUS } from './evaluation-corpus.js';
|
|
7
|
+
import { fingerprint } from './durable-store.js';
|
|
8
|
+
import { runCoreWorkflow, coreRuntimeIdentity } from './core-host.js';
|
|
9
|
+
import { ExecutorRegistry } from './executors.js';
|
|
10
|
+
import { OpenSpecTools } from './openspec.js';
|
|
11
|
+
import { validatePipelineContext } from '../installer/runtime/pipeline-state.js';
|
|
12
|
+
import { runtimeEfficiency } from './efficiency.js';
|
|
13
|
+
function oracle(test, file) {
|
|
14
|
+
return spawnSync(process.execPath, ['-e', 'const assert = require("node:assert/strict"); const api = require(process.argv[1]);' + test.oracle, file], { encoding: 'utf8', timeout: 10000 }).status === 0;
|
|
15
|
+
}
|
|
16
|
+
function prepare(test, root) {
|
|
17
|
+
const repositories = test.repositories.map(id => {
|
|
18
|
+
const directory = path.join(root, id);
|
|
19
|
+
mkdirSync(directory);
|
|
20
|
+
writeFileSync(path.join(directory, 'implementation.cjs'), test.source);
|
|
21
|
+
writeFileSync(path.join(directory, 'AGENTS.md'), '# Repository guidance\n' + 'Keep public behavior stable. Follow the existing implementation.cjs convention.\n'.repeat(180));
|
|
22
|
+
execFileSync('git', ['init', '-q', directory]);
|
|
23
|
+
execFileSync('git', ['-C', directory, 'add', '.']);
|
|
24
|
+
execFileSync('git', ['-C', directory, '-c', 'user.name=Evaluation', '-c', 'user.email=evaluation@example.invalid', 'commit', '-qm', 'frozen fixture']);
|
|
25
|
+
return { id, name: id, path: directory, baseSha: execFileSync('git', ['-C', directory, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).trim() };
|
|
26
|
+
});
|
|
27
|
+
return validatePipelineContext({ schemaVersion: 1, runId: randomUUID(), backlogRoot: root, artifactRoot: repositories[0].path, artifactRepositoryId: repositories[0].id, repositories, ownership: { git: 'host', backlog: 'host', worktrees: 'host' }, specs: [{ id: 1, title: test.id, description: test.description + ' Implement behavior in implementation.cjs in every selected repository.', repositoryIds: test.repositories, acceptanceCriteria: [test.description] }] });
|
|
28
|
+
}
|
|
29
|
+
function fixtureRegistry(test, context) {
|
|
30
|
+
let developers = 0, reviews = 0;
|
|
31
|
+
const sessions = new Map();
|
|
32
|
+
return new ExecutorRegistry().register('fixture', {
|
|
33
|
+
capabilities: () => ({ transport: 'offline-fixture', continuation: 'supported', effortSupport: 'unsupported', supportedEfforts: [], observedModel: false, observedEffort: false }),
|
|
34
|
+
async execute(request) {
|
|
35
|
+
const sessionId = 'fixture-' + request.role;
|
|
36
|
+
if (request.resumeSessionId && !sessions.has(request.resumeSessionId))
|
|
37
|
+
throw new Error('Fixture session was not restored');
|
|
38
|
+
const history = sessions.get(sessionId) ?? [];
|
|
39
|
+
history.push(request.prompt);
|
|
40
|
+
sessions.set(sessionId, history);
|
|
41
|
+
const tools = new OpenSpecTools(request.openspec);
|
|
42
|
+
await tools.execute({ action: 'load_skill' });
|
|
43
|
+
let output;
|
|
44
|
+
if (request.role === 'architect') {
|
|
45
|
+
await tools.execute({ action: 'new' });
|
|
46
|
+
const artifacts = [
|
|
47
|
+
['proposal', 'proposal.md', `## Why\n${test.description}\n## What Changes\n- Implement the requested behavior.\n## Capabilities\n### New Capabilities\n- \`fixture-behavior\`: ${test.description}\n### Modified Capabilities\nNone.\n## Impact\nSelected repositories.\n`],
|
|
48
|
+
['design', 'design.md', `## Context\n${test.description}\n## Decisions\nUse the existing implementation.cjs pattern and keep the public function boundary.`],
|
|
49
|
+
['specs', 'specs/fixture-behavior/spec.md', `## ADDED Requirements\n### Requirement: Requested behavior\nThe implementation SHALL satisfy the frozen behavior.\n#### Scenario: Supported input\n- **WHEN** a supported input is supplied\n- **THEN** the expected value is returned without losing required ordering or boundary behavior\n`],
|
|
50
|
+
['tasks', 'tasks.md', '## 1. Implementation\n- [ ] 1.1 Implement and test the requested behavior\n'],
|
|
51
|
+
];
|
|
52
|
+
for (const [artifact, file, content] of artifacts) {
|
|
53
|
+
await tools.execute({ action: 'instructions', artifact });
|
|
54
|
+
await tools.execute({ action: 'write_artifact', path: file, content });
|
|
55
|
+
}
|
|
56
|
+
output = { confidence: 'high', planningDepth: context.repositories.length > 1 ? 'full' : 'focused', referencePatterns: ['implementation.cjs'], riskFlags: [], verification: [] };
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
await tools.execute({ action: 'instructions', artifact: 'apply' });
|
|
60
|
+
if (request.role === 'developer') {
|
|
61
|
+
developers++;
|
|
62
|
+
for (const repo of context.repositories)
|
|
63
|
+
writeFileSync(path.join(repo.path, 'implementation.cjs'), test.correction && developers === 1 ? test.defects[0] : test.solution);
|
|
64
|
+
writeFileSync(path.join(context.artifactRoot, 'openspec/changes/evaluation-change/tasks.md'), '## 1. Implementation\n- [x] 1.1 Implement and test the requested behavior\n');
|
|
65
|
+
output = { summary: 'Implemented fixture behavior', files: ['implementation.cjs'], tests: [], verification: 'Core executes the frozen checks', incomplete: [] };
|
|
66
|
+
}
|
|
67
|
+
else {
|
|
68
|
+
reviews++;
|
|
69
|
+
const approved = !(test.correction === 'review' && reviews === 1);
|
|
70
|
+
output = { approved, summary: approved ? 'Behavior reviewed' : 'Stable order is missing', issues: approved ? [] : ['Preserve original order'], score: approved ? 95 : 50, aspects: Object.fromEntries(['type_correctness', 'pattern_adherence', 'test_coverage', 'security', 'architectural_alignment'].map(key => [key, approved ? 95 : 50])), acceptance: { criteria: [{ specId: '1', criterionIndex: 0, status: approved ? 'met' : 'blocked', evidence: ['implementation.cjs inspected against frozen behavior'] }], checks: [], findings: [] } };
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return { text: JSON.stringify(output), structuredOutput: output, sessionId, usage: { inputTokens: Math.ceil(request.prompt.length / 4), outputTokens: 100, costUsd: 0 } };
|
|
74
|
+
},
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
export async function runEvaluation(options) {
|
|
78
|
+
const repetitions = options.repetitions ?? 1;
|
|
79
|
+
if (!Number.isSafeInteger(repetitions) || repetitions < 1 || repetitions > 20)
|
|
80
|
+
throw new Error('Evaluation repetitions must be 1–20');
|
|
81
|
+
if (options.real && (!options.config || !Number.isFinite(options.maxCostUsd) || options.maxCostUsd <= 0 || Object.values(options.config.agents).some(role => !role.model)))
|
|
82
|
+
throw new Error('Real evaluation requires explicit per-role models and a positive aggregate spend limit');
|
|
83
|
+
const identity = coreRuntimeIdentity();
|
|
84
|
+
const observations = [];
|
|
85
|
+
let spend = 0, stopReason = null;
|
|
86
|
+
const root = mkdtempSync(path.join(tmpdir(), 'specrails-evaluation-'));
|
|
87
|
+
try {
|
|
88
|
+
for (let repeat = 0; repeat < repetitions && !stopReason; repeat++)
|
|
89
|
+
for (const test of EVALUATION_CORPUS) {
|
|
90
|
+
// Check the independent oracle itself against defective candidates.
|
|
91
|
+
const probe = path.join(root, 'oracle-probe.cjs');
|
|
92
|
+
writeFileSync(probe, test.solution);
|
|
93
|
+
if (!oracle(test, probe))
|
|
94
|
+
throw new Error('Acceptance oracle rejects reference solution: ' + test.id);
|
|
95
|
+
for (const defect of test.defects) {
|
|
96
|
+
writeFileSync(probe, defect);
|
|
97
|
+
if (oracle(test, probe))
|
|
98
|
+
throw new Error('Acceptance oracle admits defective variant: ' + test.id);
|
|
99
|
+
}
|
|
100
|
+
const order = options.real && Math.random() < 0.5 ? ['optimized', 'full'] : ['full', 'optimized'];
|
|
101
|
+
for (const mode of order) {
|
|
102
|
+
if (stopReason)
|
|
103
|
+
break;
|
|
104
|
+
const workspace = path.join(root, `${test.id}-${repeat}-${mode}`);
|
|
105
|
+
mkdirSync(workspace);
|
|
106
|
+
const context = prepare(test, workspace);
|
|
107
|
+
const base = options.config ?? { schemaVersion: 1, enabled: true, providers: [], agents: { architect: { provider: 'fixture', model: 'fixture' }, developer: { provider: 'fixture', model: 'fixture' }, reviewer: { provider: 'fixture', model: 'fixture' } }, verification: [] };
|
|
108
|
+
const config = { ...structuredClone(base), approvalBeforeArchive: false, efficiency: { schemaVersion: 1, contextMode: mode === 'full' ? 'full' : 'incremental', reviewMode: mode === 'full' ? 'full' : 'incremental', planning: mode === 'full' ? 'full' : 'proportional', verification: { maxConcurrency: 1 } }, limits: { ...base.limits, maxAttempts: 3, timeoutMs: 300000, ...(options.real ? { maxCostUsd: options.maxCostUsd - spend } : {}) }, verification: context.repositories.map(repo => ({ repositoryId: repo.id, command: process.execPath, args: ['-e', 'const assert = require("node:assert/strict"); const api = require("./implementation.cjs");' + (test.verification ?? '')] })) };
|
|
109
|
+
if (options.real && config.limits.maxCostUsd <= 0) {
|
|
110
|
+
stopReason = 'Aggregate spend limit reached';
|
|
111
|
+
break;
|
|
112
|
+
}
|
|
113
|
+
const started = Date.now();
|
|
114
|
+
let state;
|
|
115
|
+
try {
|
|
116
|
+
state = await runCoreWorkflow({ context, change: 'evaluation-change', config, ...(options.real ? {} : { registry: fixtureRegistry(test, context) }) });
|
|
117
|
+
}
|
|
118
|
+
catch (error) {
|
|
119
|
+
observations.push({ caseId: test.id, repeat, mode, status: 'blocked', independentAccepted: false, error: error instanceof Error ? error.message : String(error), runtimeIdentity: identity, taskHash: fingerprint(test), oracleHash: fingerprint(test.oracle), repositories: context.repositories.map(({ id, baseSha }) => ({ id, baseSha })), configHash: fingerprint(config) });
|
|
120
|
+
stopReason = 'Runtime preflight failed; experiment stopped without retry';
|
|
121
|
+
break;
|
|
122
|
+
}
|
|
123
|
+
const accepted = context.repositories.every(repo => oracle(test, path.join(repo.path, 'implementation.cjs')));
|
|
124
|
+
const metrics = runtimeEfficiency(state);
|
|
125
|
+
const calls = state.history.flatMap(attempt => attempt.invocations ?? []);
|
|
126
|
+
observations.push({ caseId: test.id, repeat, mode, order, taskHash: fingerprint(test), oracleHash: fingerprint(test.oracle), repositories: context.repositories.map(({ id, baseSha }) => ({ id, baseSha })), configHash: fingerprint(config), runtimeIdentity: identity, cacheState: 'uncontrolled-provider-cache; fresh workspace and role sessions', status: state.status, independentAccepted: accepted && state.status === 'succeeded', activeDurationMs: Date.now() - started, metrics, invocations: calls.map(call => ({ kind: call.kind, promptBytes: call.promptBytes, provider: call.provider, model: call.model, status: call.status })), failures: state.history.filter(attempt => attempt.status !== 'succeeded' || attempt.output?.valid === false || attempt.output?.approved === false).map(attempt => ({ role: attempt.stepId, status: attempt.status, error: attempt.error ?? null })) });
|
|
127
|
+
if (options.real) {
|
|
128
|
+
if (metrics.total.costUsd === null)
|
|
129
|
+
stopReason = 'Billing incomplete; monetary comparison is inconclusive and further spend is stopped';
|
|
130
|
+
else
|
|
131
|
+
spend += metrics.total.costUsd;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
if (stopReason)
|
|
135
|
+
break;
|
|
136
|
+
}
|
|
137
|
+
const median = (values) => { const sorted = [...values].sort((a, b) => a - b); return sorted.length ? (sorted[Math.floor((sorted.length - 1) / 2)] + sorted[Math.floor(sorted.length / 2)]) / 2 : null; };
|
|
138
|
+
const groups = Object.fromEntries(['full', 'optimized'].map(mode => {
|
|
139
|
+
const rows = observations.filter(row => row.mode === mode);
|
|
140
|
+
const accepted = rows.filter(row => row.independentAccepted).length;
|
|
141
|
+
const costs = rows.map(row => row.metrics?.total.costUsd);
|
|
142
|
+
const knownCost = costs.every(value => typeof value === 'number') ? costs.reduce((sum, value) => sum + value, 0) : null;
|
|
143
|
+
const durations = rows.flatMap(row => typeof row.activeDurationMs === 'number' ? [row.activeDurationMs] : []);
|
|
144
|
+
const average = durations.length ? durations.reduce((sum, value) => sum + value, 0) / durations.length : null;
|
|
145
|
+
return [mode, { samples: rows.length, independentlyAccepted: accepted, costPerAcceptedUsd: options.real && knownCost !== null && accepted ? knownCost / accepted : null, medianActiveDurationMs: median(durations), minActiveDurationMs: durations.length ? Math.min(...durations) : null, maxActiveDurationMs: durations.length ? Math.max(...durations) : null, standardDeviationMs: average === null ? null : Math.sqrt(durations.reduce((sum, value) => sum + (value - average) ** 2, 0) / durations.length) }];
|
|
146
|
+
}));
|
|
147
|
+
const full = groups.full, optimized = groups.optimized;
|
|
148
|
+
const paired = observations.length === EVALUATION_CORPUS.length * repetitions * 2 && full.samples === optimized.samples;
|
|
149
|
+
const noObservedQualityDrop = paired && optimized.independentlyAccepted >= full.independentlyAccepted;
|
|
150
|
+
const promptPairs = observations.filter(row => row.mode === 'full' && row.caseId === 'verification-correction').map(row => {
|
|
151
|
+
const other = observations.find(item => item.mode === 'optimized' && item.caseId === row.caseId && item.repeat === row.repeat);
|
|
152
|
+
const correction = (value) => value?.invocations?.find(call => call.kind === 'correction')?.promptBytes;
|
|
153
|
+
const before = correction(row), after = correction(other);
|
|
154
|
+
return { caseId: row.caseId, repeat: row.repeat, fullBytes: before ?? null, optimizedBytes: after ?? null, reduction: before && after !== undefined ? 1 - after / before : null };
|
|
155
|
+
});
|
|
156
|
+
const noExtraInvocations = paired && observations.filter(row => row.mode === 'full').every(row => {
|
|
157
|
+
const other = observations.find(item => item.mode === 'optimized' && item.caseId === row.caseId && item.repeat === row.repeat);
|
|
158
|
+
return Array.isArray(row.invocations) && Array.isArray(other?.invocations) && other.invocations.length <= row.invocations.length;
|
|
159
|
+
});
|
|
160
|
+
const monetaryConclusion = options.real && paired && full.costPerAcceptedUsd !== null && optimized.costPerAcceptedUsd !== null && full.costPerAcceptedUsd > 0 ? optimized.costPerAcceptedUsd <= full.costPerAcceptedUsd * 0.8 && noObservedQualityDrop ? 'target-met-in-this-sample' : 'target-not-met-in-this-sample' : 'inconclusive';
|
|
161
|
+
const report = { schemaVersion: 1, experiment: Object.values(options.config?.agents ?? {}).some(role => role.escalation) ? 'routing' : 'same-model', noExtraInvocations, mode: options.real ? 'real' : 'offline', runtimeIdentity: identity, corpusHash: fingerprint(EVALUATION_CORPUS), measurement: options.real ? 'reported-provider-usage' : 'synthetic-fixture-usage; no actual AI savings measured', monetaryTarget: 0.2, monetaryConclusion, groups, noObservedQualityDrop, promptPairs, correctionPromptTargetMet: promptPairs.length > 0 && promptPairs.every(pair => pair.reduction !== null && pair.reduction >= 0.4), sampleLimitations: 'Small descriptive paired sample; does not establish a universal quality or savings guarantee.', stopReason, reportedSpendUsd: options.real ? spend : 0, observations };
|
|
162
|
+
mkdirSync(options.output, { recursive: true });
|
|
163
|
+
writeFileSync(path.join(options.output, 'evaluation.json'), JSON.stringify(report, null, 2) + '\n');
|
|
164
|
+
writeFileSync(path.join(options.output, 'evaluation.md'), `# Implementation efficiency evaluation\n\nMode: ${report.mode}. ${report.measurement}.\n\nMonetary target: 20% lower aggregate cost per independently accepted output. Conclusion: ${monetaryConclusion}.\n\n${JSON.stringify(groups, null, 2)}\n\nCorrection prompt comparison: ${JSON.stringify(promptPairs)}\n\n${observations.map(row => `- ${row.caseId}, ${row.mode}: ${row.status}; independent acceptance ${row.independentAccepted}; ${row.activeDurationMs} ms`).join('\n')}\n\n${stopReason ?? (options.real ? report.sampleLimitations : 'No paid benchmark has established monetary savings.')}\n`);
|
|
165
|
+
return report;
|
|
166
|
+
}
|
|
167
|
+
finally {
|
|
168
|
+
rmSync(root, { recursive: true, force: true });
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
//# sourceMappingURL=evaluation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"evaluation.js","sourceRoot":"","sources":["../../src/agent-runtime/evaluation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAA;AAC5D,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AACvE,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAA;AAChC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,iBAAiB,EAAuB,MAAM,wBAAwB,CAAA;AAC/E,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAA;AAChD,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAA;AACrE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,uBAAuB,EAAwB,MAAM,wCAAwC,CAAA;AAEtG,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAGnD,SAAS,MAAM,CAAC,IAAoB,EAAE,IAAY;IAChD,OAAO,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,qFAAqF,GAAG,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC,CAAA;AAC1M,CAAC;AACD,SAAS,OAAO,CAAC,IAAoB,EAAE,IAAY;IACjD,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;QAC9C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QAC3D,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,oBAAoB,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;QACtE,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,EAAE,yBAAyB,GAAG,mFAAmF,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;QAC7K,YAAY,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,CAAA;QAC9C,YAAY,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,CAAA;QAClD,YAAY,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,sBAAsB,EAAE,IAAI,EAAE,uCAAuC,EAAE,QAAQ,EAAE,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAA;QACtJ,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,YAAY,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAA;IAC7I,CAAC,CAAC,CAAA;IACF,OAAO,uBAAuB,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,YAAY,EAAE,YAAY,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,oBAAoB,EAAE,YAAY,CAAC,CAAC,CAAE,CAAC,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,GAAG,yEAAyE,EAAE,aAAa,EAAE,IAAI,CAAC,YAAY,EAAE,kBAAkB,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,CAAC,CAAA;AACrd,CAAC;AACD,SAAS,eAAe,CAAC,IAAoB,EAAE,OAAwB;IACrE,IAAI,UAAU,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,CAAA;IAC/B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAoB,CAAA;IAC5C,OAAO,IAAI,gBAAgB,EAAE,CAAC,QAAQ,CAAC,SAAS,EAAE;QAChD,YAAY,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,iBAAiB,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,aAAa,EAAE,gBAAgB,EAAE,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;QAClL,KAAK,CAAC,OAAO,CAAC,OAAqB;YACjC,MAAM,SAAS,GAAG,UAAU,GAAG,OAAO,CAAC,IAAI,CAAA;YAC3C,IAAI,OAAO,CAAC,eAAe,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,eAAe,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;YAC1H,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;YAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;YAC7G,MAAM,KAAK,GAAG,IAAI,aAAa,CAAC,OAAO,CAAC,QAAS,CAAC,CAAA;YAClD,MAAM,KAAK,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAA;YAC7C,IAAI,MAAc,CAAA;YAClB,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBACjC,MAAM,KAAK,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAA;gBACtC,MAAM,SAAS,GAAG;oBAChB,CAAC,UAAU,EAAE,aAAa,EAAE,WAAW,IAAI,CAAC,WAAW,0HAA0H,IAAI,CAAC,WAAW,yEAAyE,CAAC;oBAC3Q,CAAC,QAAQ,EAAE,WAAW,EAAE,eAAe,IAAI,CAAC,WAAW,oGAAoG,CAAC;oBAC5J,CAAC,OAAO,EAAE,gCAAgC,EAAE,gSAAgS,CAAC;oBAC7U,CAAC,OAAO,EAAE,UAAU,EAAE,6EAA6E,CAAC;iBACrG,CAAA;gBACD,KAAK,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,SAAS,EAAE,CAAC;oBAAC,MAAM,KAAK,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC,CAAC;oBAAC,MAAM,KAAK,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,gBAAgB,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAA;gBAAC,CAAC;gBACxL,MAAM,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,EAAE,iBAAiB,EAAE,CAAC,oBAAoB,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,CAAA;YAClL,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAA;gBAClE,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;oBACjC,UAAU,EAAE,CAAA;oBACZ,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,YAAY;wBAAE,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,oBAAoB,CAAC,EAAE,IAAI,CAAC,UAAU,IAAI,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;oBAC1K,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,6CAA6C,CAAC,EAAE,6EAA6E,CAAC,CAAA;oBAC5K,MAAM,GAAG,EAAE,OAAO,EAAE,8BAA8B,EAAE,KAAK,EAAE,CAAC,oBAAoB,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,YAAY,EAAE,iCAAiC,EAAE,UAAU,EAAE,EAAE,EAAE,CAAA;gBACjK,CAAC;qBAAM,CAAC;oBACN,OAAO,EAAE,CAAA;oBACT,MAAM,QAAQ,GAAG,CAAC,CAAC,IAAI,CAAC,UAAU,KAAK,QAAQ,IAAI,OAAO,KAAK,CAAC,CAAC,CAAA;oBACjE,MAAM,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,yBAAyB,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,yBAAyB,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,kBAAkB,EAAE,mBAAmB,EAAE,eAAe,EAAE,UAAU,EAAE,yBAAyB,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,cAAc,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,EAAE,QAAQ,EAAE,CAAC,sDAAsD,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAA;gBACthB,CAAC;YACH,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,gBAAgB,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,YAAY,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE,CAAA;QAC3K,CAAC;KACF,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAA0B;IAC5D,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAA;IAC5C,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,IAAI,WAAW,GAAG,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAA;IACrI,IAAI,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,UAAW,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,wFAAwF,CAAC,CAAA;IACtR,MAAM,QAAQ,GAAG,mBAAmB,EAAE,CAAA;IACtC,MAAM,YAAY,GAAmC,EAAE,CAAA;IACvD,IAAI,KAAK,GAAG,CAAC,EAAE,UAAU,GAAkB,IAAI,CAAA;IAC/C,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,uBAAuB,CAAC,CAAC,CAAA;IACtE,IAAI,CAAC;QACH,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,WAAW,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE;YAAE,KAAK,MAAM,IAAI,IAAI,iBAAiB,EAAE,CAAC;gBACzG,oEAAoE;gBACpE,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAA;gBACjD,aAAa,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAA;gBACnC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,GAAG,IAAI,CAAC,EAAE,CAAC,CAAA;gBACrG,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;oBAAC,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;oBAAC,IAAI,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC;wBAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,GAAG,IAAI,CAAC,EAAE,CAAC,CAAA;gBAAC,CAAC;gBACvK,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC,CAAA;gBACjG,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,IAAI,UAAU;wBAAE,MAAK;oBACrB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,EAAE,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC,CAAC;oBAAC,SAAS,CAAC,SAAS,CAAC,CAAA;oBACvF,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;oBACxC,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,aAAa,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,SAAS,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,QAAQ,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,CAAA;oBAChR,MAAM,MAAM,GAAkB,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,EAAE,qBAAqB,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,aAAa,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,aAAa,EAAE,UAAU,EAAE,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,aAAa,EAAE,QAAQ,EAAE,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc,EAAE,YAAY,EAAE,EAAE,cAAc,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAW,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,4FAA4F,GAAG,CAAC,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAA;oBACtrB,IAAI,OAAO,CAAC,IAAI,IAAI,MAAM,CAAC,MAAO,CAAC,UAAW,IAAI,CAAC,EAAE,CAAC;wBAAC,UAAU,GAAG,+BAA+B,CAAC;wBAAC,MAAK;oBAAC,CAAC;oBAC5G,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;oBAC1B,IAAI,KAAK,CAAA;oBACT,IAAI,CAAC;wBAAC,KAAK,GAAG,MAAM,eAAe,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,mBAAmB,EAAE,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,eAAe,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,CAAA;oBAAC,CAAC;oBAC9J,OAAO,KAAK,EAAE,CAAC;wBAAC,YAAY,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,mBAAmB,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,EAAE,UAAU,EAAE,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;wBAAC,UAAU,GAAG,4DAA4D,CAAC;wBAAC,MAAK;oBAAC,CAAC;oBACrd,MAAM,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC,CAAC,CAAA;oBAC7G,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAA;oBACxC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC,CAAA;oBACzE,YAAY,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,EAAE,UAAU,EAAE,WAAW,CAAC,MAAM,CAAC,EAAE,eAAe,EAAE,QAAQ,EAAE,UAAU,EAAE,gEAAgE,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,mBAAmB,EAAE,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,WAAW,EAAE,gBAAgB,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,KAAK,WAAW,IAAK,OAAO,CAAC,MAA8D,EAAE,KAAK,KAAK,KAAK,IAAK,OAAO,CAAC,MAA6C,EAAE,QAAQ,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAA;oBAC38B,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;wBACjB,IAAI,OAAO,CAAC,KAAK,CAAC,OAAO,KAAK,IAAI;4BAAE,UAAU,GAAG,sFAAsF,CAAA;;4BAClI,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC,OAAO,CAAA;oBACrC,CAAC;gBACH,CAAC;gBACD,IAAI,UAAU;oBAAE,MAAK;YACvB,CAAC;QACD,MAAM,MAAM,GAAG,CAAC,MAAgB,EAAE,EAAE,GAAG,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAE,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA,CAAC,CAAC,CAAA;QACpN,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;YACjE,MAAM,IAAI,GAAG,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAA;YAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,MAAM,CAAA;YACnE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAE,GAAG,CAAC,OAA4D,EAAE,KAAK,CAAC,OAAO,CAAC,CAAA;YAC/G,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAS,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;YAChI,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,GAAG,CAAC,gBAAgB,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAC7G,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAA;YAC7G,OAAO,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,qBAAqB,EAAE,QAAQ,EAAE,kBAAkB,EAAE,OAAO,CAAC,IAAI,IAAI,SAAS,KAAK,IAAI,IAAI,QAAQ,CAAC,CAAC,CAAC,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,sBAAsB,EAAE,MAAM,CAAC,SAAS,CAAC,EAAE,mBAAmB,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QACrf,CAAC,CAAC,CAAC,CAAA;QACH,MAAM,IAAI,GAAG,MAAM,CAAC,IAAK,EAAE,SAAS,GAAG,MAAM,CAAC,SAAU,CAAA;QACxD,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,KAAK,iBAAiB,CAAC,MAAM,GAAG,WAAW,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,CAAC,OAAO,CAAA;QACvH,MAAM,qBAAqB,GAAG,MAAM,IAAI,SAAS,CAAC,qBAAqB,IAAI,IAAI,CAAC,qBAAqB,CAAA;QACrG,MAAM,WAAW,GAAG,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK,yBAAyB,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACxH,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,IAAI,IAAI,CAAC,MAAM,KAAK,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,CAAA;YAC9H,MAAM,UAAU,GAAG,CAAC,KAA0C,EAAE,EAAE,CAAE,KAAK,EAAE,WAA0E,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,EAAE,WAAW,CAAA;YAC5M,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,EAAE,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;YACzD,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,IAAI,IAAI,EAAE,cAAc,EAAE,KAAK,IAAI,IAAI,EAAE,SAAS,EAAE,MAAM,IAAI,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;QACnL,CAAC,CAAC,CAAA;QACF,MAAM,kBAAkB,GAAG,MAAM,IAAI,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE;YAC/F,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,IAAI,IAAI,CAAC,MAAM,KAAK,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,CAAA;YAC9H,OAAO,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,WAAW,CAAC,IAAI,KAAK,CAAC,WAAW,CAAC,MAAM,IAAI,GAAG,CAAC,WAAW,CAAC,MAAM,CAAA;QAClI,CAAC,CAAC,CAAA;QACF,MAAM,kBAAkB,GAAG,OAAO,CAAC,IAAI,IAAI,MAAM,IAAI,IAAI,CAAC,kBAAkB,KAAK,IAAI,IAAI,SAAS,CAAC,kBAAkB,KAAK,IAAI,IAAI,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,kBAAkB,IAAI,IAAI,CAAC,kBAAkB,GAAG,GAAG,IAAI,qBAAqB,CAAC,CAAC,CAAC,2BAA2B,CAAC,CAAC,CAAC,+BAA+B,CAAC,CAAC,CAAC,cAAc,CAAA;QACvU,MAAM,MAAM,GAAG,EAAE,aAAa,EAAE,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,EAAE,kBAAkB,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,EAAE,eAAe,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,CAAC,iBAAiB,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,wDAAwD,EAAE,cAAc,EAAE,GAAG,EAAE,kBAAkB,EAAE,MAAM,EAAE,qBAAqB,EAAE,WAAW,EAAE,yBAAyB,EAAE,WAAW,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,KAAK,IAAI,IAAI,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC,EAAE,iBAAiB,EAAE,+FAA+F,EAAE,UAAU,EAAE,gBAAgB,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,CAAA;QAC5xB,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QAC9C,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,iBAAiB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;QACnG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,eAAe,CAAC,EAAE,mDAAmD,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,WAAW,iGAAiG,kBAAkB,QAAQ,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,qCAAqC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,OAAO,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,GAAG,CAAC,MAAM,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,MAAM,4BAA4B,GAAG,CAAC,mBAAmB,KAAK,GAAG,CAAC,gBAAgB,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,UAAU,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,qDAAqD,CAAC,IAAI,CAAC,CAAA;QAC3oB,OAAO,MAAM,CAAA;IACf,CAAC;YAAS,CAAC;QAAC,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAAC,CAAC;AAC9D,CAAC"}
|