veryfront 0.1.923 → 0.1.925
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/cli/commands/workflow/command.d.ts +1 -0
- package/esm/cli/commands/workflow/command.d.ts.map +1 -1
- package/esm/cli/commands/workflow/command.js +41 -16
- package/esm/deno.js +1 -1
- package/esm/src/eval/agent-service.d.ts.map +1 -1
- package/esm/src/eval/agent-service.js +86 -16
- package/esm/src/eval/expect.d.ts.map +1 -1
- package/esm/src/eval/expect.js +23 -1
- package/esm/src/eval/index.d.ts +2 -1
- package/esm/src/eval/index.d.ts.map +1 -1
- package/esm/src/eval/index.js +1 -0
- package/esm/src/eval/metrics.d.ts +4 -1
- package/esm/src/eval/metrics.d.ts.map +1 -1
- package/esm/src/eval/metrics.js +26 -4
- package/esm/src/eval/tool-behavior.d.ts +12 -0
- package/esm/src/eval/tool-behavior.d.ts.map +1 -0
- package/esm/src/eval/tool-behavior.js +104 -0
- package/esm/src/eval/types.d.ts +22 -1
- package/esm/src/eval/types.d.ts.map +1 -1
- package/esm/src/server/handlers/request/agent-stream.handler.d.ts.map +1 -1
- package/esm/src/server/handlers/request/agent-stream.handler.js +53 -1
- package/esm/src/utils/version-constant.d.ts +1 -1
- package/esm/src/utils/version-constant.js +1 -1
- package/esm/src/workflow/worker/dynamic-run-entrypoint.d.ts.map +1 -1
- package/esm/src/workflow/worker/dynamic-run-entrypoint.js +16 -10
- package/esm/src/workflow/worker/run-entrypoint.d.ts.map +1 -1
- package/esm/src/workflow/worker/run-entrypoint.js +6 -0
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/workflow/command.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/workflow/command.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAKjD,MAAM,WAAW,eAAgB,SAAQ,YAAY;IACnD,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,wBAAgB,6BAA6B,CAC3C,MAAM,EAAE,sBAAsB,EAAE,GAC/B,MAAM,EAAE,CAUV;AAuFD,wBAAsB,eAAe,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAyF7E"}
|
|
@@ -2,6 +2,9 @@ import * as dntShim from "../../../_dnt.shims.js";
|
|
|
2
2
|
import { cliLogger } from "../../utils/index.js";
|
|
3
3
|
import { exitProcess } from "../../utils/index.js";
|
|
4
4
|
import { withProjectSourceContext } from "../../shared/project-source-context.js";
|
|
5
|
+
import { agentRegistry } from "../../../src/agent/composition/index.js";
|
|
6
|
+
import { discoverProjectAgentRuntime } from "../../../src/agent/project/agent-runtime.js";
|
|
7
|
+
import { toolRegistry } from "../../../src/tool/registry.js";
|
|
5
8
|
import { sanitizeRunOutputForLogging } from "../../utils/sanitize-run-output.js";
|
|
6
9
|
import { writeRunResultIfConfigured } from "../../utils/write-run-result.js";
|
|
7
10
|
import { getEnv } from "../../../src/platform/index.js";
|
|
@@ -16,18 +19,39 @@ export function formatWorkflowDiscoveryErrors(errors) {
|
|
|
16
19
|
}
|
|
17
20
|
return lines;
|
|
18
21
|
}
|
|
19
|
-
|
|
22
|
+
function formatRuntimeDiscoveryError(error) {
|
|
23
|
+
return {
|
|
24
|
+
filePath: error.file,
|
|
25
|
+
error: error.error.message,
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
function withProjectStepRegistries(config) {
|
|
29
|
+
return {
|
|
30
|
+
...config,
|
|
31
|
+
executor: {
|
|
32
|
+
...config.executor,
|
|
33
|
+
stepExecutor: {
|
|
34
|
+
...config.executor?.stepExecutor,
|
|
35
|
+
agentRegistry: config.executor?.stepExecutor?.agentRegistry ?? agentRegistry,
|
|
36
|
+
toolRegistry: config.executor?.stepExecutor?.toolRegistry ?? toolRegistry,
|
|
37
|
+
},
|
|
38
|
+
},
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
async function createWorkflowClient(config) {
|
|
20
42
|
const { createWorkflowClient } = await import("../../../src/workflow/api/workflow-client.js");
|
|
43
|
+
const clientConfig = withProjectStepRegistries(config);
|
|
21
44
|
const redisUrl = getEnv("REDIS_URL")?.trim();
|
|
22
45
|
if (!redisUrl) {
|
|
23
|
-
return createWorkflowClient(
|
|
46
|
+
return createWorkflowClient(clientConfig);
|
|
24
47
|
}
|
|
25
48
|
const { RedisBackend } = await import("../../../src/workflow/backends/redis.js");
|
|
49
|
+
const debug = clientConfig.debug ?? false;
|
|
26
50
|
const backend = new RedisBackend({ url: redisUrl, debug });
|
|
27
51
|
if (backend.initialize) {
|
|
28
52
|
await backend.initialize();
|
|
29
53
|
}
|
|
30
|
-
return createWorkflowClient({
|
|
54
|
+
return createWorkflowClient({ ...clientConfig, backend });
|
|
31
55
|
}
|
|
32
56
|
async function waitForWorkflowExit(client, runId) {
|
|
33
57
|
while (true) {
|
|
@@ -79,37 +103,38 @@ export async function workflowCommand(options) {
|
|
|
79
103
|
return;
|
|
80
104
|
}
|
|
81
105
|
}
|
|
82
|
-
const
|
|
83
|
-
await withProjectSourceContext(
|
|
106
|
+
const projectDir = options.projectDir ?? dntShim.Deno.cwd();
|
|
107
|
+
await withProjectSourceContext(projectDir, async ({ adapter, proxyContext }) => {
|
|
84
108
|
const sourceLabel = proxyContext?.branchRef
|
|
85
109
|
? `branch ${proxyContext.branchRef}`
|
|
86
110
|
: proxyContext
|
|
87
111
|
? "main"
|
|
88
|
-
: `${
|
|
112
|
+
: `${projectDir}/workflows/...`;
|
|
89
113
|
cliLogger.info(`Discovering workflows in ${sourceLabel}`);
|
|
90
|
-
const discovery = await
|
|
91
|
-
projectDir
|
|
114
|
+
const discovery = await discoverProjectAgentRuntime({
|
|
115
|
+
projectDir,
|
|
92
116
|
adapter,
|
|
93
|
-
|
|
94
|
-
debug: options.debug,
|
|
117
|
+
verbose: options.debug,
|
|
95
118
|
});
|
|
119
|
+
const workflows = [...discovery.workflows.values()];
|
|
96
120
|
if (discovery.errors.length > 0 && options.debug) {
|
|
97
|
-
for (const err of discovery.errors) {
|
|
121
|
+
for (const err of discovery.errors.map(formatRuntimeDiscoveryError)) {
|
|
98
122
|
cliLogger.warn(` Warning: ${err.filePath}: ${err.error}`);
|
|
99
123
|
}
|
|
100
124
|
}
|
|
101
|
-
const workflow =
|
|
125
|
+
const workflow = workflows.find((candidate) => candidate.id === workflowId);
|
|
102
126
|
if (!workflow) {
|
|
103
127
|
cliLogger.error(`Workflow "${workflowId}" not found.`);
|
|
104
128
|
if (discovery.errors.length > 0 && !options.debug) {
|
|
105
129
|
cliLogger.warn("Some workflow files could not be loaded:");
|
|
106
|
-
|
|
130
|
+
const errors = discovery.errors.map(formatRuntimeDiscoveryError);
|
|
131
|
+
for (const line of formatWorkflowDiscoveryErrors(errors)) {
|
|
107
132
|
cliLogger.warn(line);
|
|
108
133
|
}
|
|
109
134
|
}
|
|
110
|
-
if (
|
|
135
|
+
if (workflows.length > 0) {
|
|
111
136
|
cliLogger.info("Available workflows:");
|
|
112
|
-
for (const candidate of
|
|
137
|
+
for (const candidate of workflows) {
|
|
113
138
|
cliLogger.info(` - ${candidate.id}`);
|
|
114
139
|
}
|
|
115
140
|
}
|
|
@@ -119,7 +144,7 @@ export async function workflowCommand(options) {
|
|
|
119
144
|
exitProcess(1);
|
|
120
145
|
return;
|
|
121
146
|
}
|
|
122
|
-
const client = await createWorkflowClient(options.debug);
|
|
147
|
+
const client = await createWorkflowClient({ debug: options.debug });
|
|
123
148
|
try {
|
|
124
149
|
client.register(workflow.definition);
|
|
125
150
|
cliLogger.info(`Running workflow: ${workflow.id}`);
|
package/esm/deno.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agent-service.d.ts","sourceRoot":"","sources":["../../../src/src/eval/agent-service.ts"],"names":[],"mappings":"AAAA,OAAO,yBAAyB,CAAC;AAEjC,OAAO,EAEL,KAAK,uBAAuB,
|
|
1
|
+
{"version":3,"file":"agent-service.d.ts","sourceRoot":"","sources":["../../../src/src/eval/agent-service.ts"],"names":[],"mappings":"AAAA,OAAO,yBAAyB,CAAC;AAEjC,OAAO,EAEL,KAAK,uBAAuB,EAM7B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EACV,gBAAgB,EAChB,uBAAuB,EAIxB,MAAM,YAAY,CAAC;AAEpB,cAAc,qCAAqC,CAAC;AACpD,cAAc,+CAA+C,CAAC;AAE9D,gEAAgE;AAChE,eAAO,MAAM,mCAAmC,oCAAoC,CAAC;AAErF,gEAAgE;AAChE,MAAM,MAAM,gCAAgC,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC,CAAC;AAE3F,gEAAgE;AAChE,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,kEAAkE;AAClE,MAAM,WAAW,0CAA0C;IACzD,EAAE,EAAE,OAAO,CAAC;IACZ,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,mEAAmE;AACnE,MAAM,WAAW,8BAA8B;IAC7C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gBAAgB,CAAC,EAAE;QACjB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;QACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AAED,4DAA4D;AAC5D,MAAM,WAAW,qCAAqC;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,OAAO,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,4DAA4D;AAC5D,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,KAAK,EAAE,EAAE,CAAC;IACV,OAAO,EAAE,EAAE,CAAC;IACZ,cAAc,CAAC,EAAE;QACf,SAAS,EAAE,8BAA8B,CAAC;KAC3C,CAAC;IACF,QAAQ,EAAE,KAAK,CAAC;QACd,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,KAAK,CAAC;YACX,IAAI,EAAE,MAAM,CAAC;YACb,IAAI,EAAE,MAAM,CAAC;SACd,CAAC,CAAC;KACJ,CAAC,CAAC;CACJ;AAED,6DAA6D;AAC7D,MAAM,WAAW,6BAA6B;IAC5C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,GAAG,GAAG,GAAG,OAAO,EAAE,IAAI,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;IACjF,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,uBAAuB,EAAE,OAAO,EAAE,uBAAuB,KAAK,IAAI,CAAC;CAC5F;AAuSD,wEAAwE;AACxE,wBAAgB,kCAAkC,CAChD,GAAG,GAAE,gCAAqC,GACzC,2BAA2B,CAoB7B;AAED,oFAAoF;AACpF,wBAAgB,mCAAmC,CACjD,GAAG,GAAE,gCAAqC,EAC1C,cAAc,SAAiD,GAC9D,0CAA0C,CAe5C;AAED,8DAA8D;AAC9D,wBAAgB,gCAAgC,CAC9C,KAAK,EAAE,qCAAqC,GAC3C,2BAA2B,CAyB7B;AAED,mGAAmG;AACnG,wBAAgB,6BAA6B,CAC3C,MAAM,EAAE,6BAA6B,GACpC,gBAAgB,CAuElB"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import "../../_dnt.polyfills.js";
|
|
2
2
|
import * as dntShim from "../../_dnt.shims.js";
|
|
3
|
-
import { agUiSseEventTypes, getAgUiSseStringField, parseAgentServiceConfig, parseAgUiSseResponse, } from "../agent/index.js";
|
|
3
|
+
import { agUiSseEventTypes, getAgUiSseStringField, mergeToolInputDelta, parseAgentServiceConfig, parseAgUiSseResponse, } from "../agent/index.js";
|
|
4
4
|
export * from "./agent-service/live-evals/index.js";
|
|
5
5
|
export * from "./agent-service/durable-run-canaries/index.js";
|
|
6
6
|
/** Default local AG-UI endpoint used by agent-service evals. */
|
|
@@ -109,6 +109,50 @@ function stringifyError(value) {
|
|
|
109
109
|
function getToolResultError(event) {
|
|
110
110
|
return stringifyError(event.result) ?? stringifyError(event.content) ?? "Tool call failed";
|
|
111
111
|
}
|
|
112
|
+
function parseJsonString(value) {
|
|
113
|
+
try {
|
|
114
|
+
return JSON.parse(value);
|
|
115
|
+
}
|
|
116
|
+
catch {
|
|
117
|
+
return value;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
function readToolInputDelta(event) {
|
|
121
|
+
return readString(event.delta) ?? readString(event.inputTextDelta) ?? readString(event.argsDelta);
|
|
122
|
+
}
|
|
123
|
+
function readToolOutput(event) {
|
|
124
|
+
if (Object.hasOwn(event, "result"))
|
|
125
|
+
return event.result;
|
|
126
|
+
if (Object.hasOwn(event, "output"))
|
|
127
|
+
return event.output;
|
|
128
|
+
if (Object.hasOwn(event, "content")) {
|
|
129
|
+
return typeof event.content === "string" ? parseJsonString(event.content) : event.content;
|
|
130
|
+
}
|
|
131
|
+
return undefined;
|
|
132
|
+
}
|
|
133
|
+
function isDeniedToolResult(event, error) {
|
|
134
|
+
const status = getAgUiSseStringField(event, "status");
|
|
135
|
+
return status === "denied" || error?.toLowerCase().includes("denied") === true;
|
|
136
|
+
}
|
|
137
|
+
function getToolCallEntry(toolCalls, key, id, name) {
|
|
138
|
+
const existing = toolCalls.get(key);
|
|
139
|
+
if (existing) {
|
|
140
|
+
if (id && !existing.call.id)
|
|
141
|
+
existing.call.id = id;
|
|
142
|
+
if (name && existing.call.name === "tool")
|
|
143
|
+
existing.call.name = name;
|
|
144
|
+
return existing;
|
|
145
|
+
}
|
|
146
|
+
const next = {
|
|
147
|
+
call: {
|
|
148
|
+
...(id ? { id } : {}),
|
|
149
|
+
name: name ?? "tool",
|
|
150
|
+
status: "ok",
|
|
151
|
+
},
|
|
152
|
+
};
|
|
153
|
+
toolCalls.set(key, next);
|
|
154
|
+
return next;
|
|
155
|
+
}
|
|
112
156
|
function createToolCalls(events) {
|
|
113
157
|
const toolCalls = new Map();
|
|
114
158
|
for (const [index, event] of events.entries()) {
|
|
@@ -119,30 +163,56 @@ function createToolCalls(events) {
|
|
|
119
163
|
continue;
|
|
120
164
|
const id = getAgUiSseStringField(event, "toolCallId");
|
|
121
165
|
const key = id ?? `name:${name}`;
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
});
|
|
166
|
+
getToolCallEntry(toolCalls, key, id, name);
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
if (type === agUiSseEventTypes.toolCallArgs) {
|
|
170
|
+
const id = getAgUiSseStringField(event, "toolCallId");
|
|
171
|
+
const toolName = getAgUiSseStringField(event, "toolCallName");
|
|
172
|
+
const key = id ?? (toolName ? `name:${toolName}` : `args:${index}`);
|
|
173
|
+
const entry = getToolCallEntry(toolCalls, key, id, toolName);
|
|
174
|
+
const input = Object.hasOwn(event, "input") ? event.input : undefined;
|
|
175
|
+
if (input !== undefined) {
|
|
176
|
+
entry.call.input = input;
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
const delta = readToolInputDelta(event);
|
|
180
|
+
if (delta === undefined)
|
|
181
|
+
continue;
|
|
182
|
+
entry.inputText = mergeToolInputDelta(entry.inputText ?? "", delta);
|
|
183
|
+
entry.call.input = parseJsonString(entry.inputText);
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
if (type === agUiSseEventTypes.toolCallEnd) {
|
|
187
|
+
const id = getAgUiSseStringField(event, "toolCallId");
|
|
188
|
+
const toolName = getAgUiSseStringField(event, "toolCallName");
|
|
189
|
+
const key = id ?? (toolName ? `name:${toolName}` : `end:${index}`);
|
|
190
|
+
getToolCallEntry(toolCalls, key, id, toolName);
|
|
129
191
|
continue;
|
|
130
192
|
}
|
|
131
193
|
if (type === agUiSseEventTypes.toolCallResult) {
|
|
132
194
|
const id = getAgUiSseStringField(event, "toolCallId");
|
|
133
195
|
const toolName = getAgUiSseStringField(event, "toolCallName");
|
|
134
196
|
const key = id ?? (toolName ? `name:${toolName}` : `result:${index}`);
|
|
135
|
-
const
|
|
197
|
+
const entry = getToolCallEntry(toolCalls, key, id, toolName);
|
|
136
198
|
const failed = event.isError === true;
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
199
|
+
const error = failed ? getToolResultError(event) : undefined;
|
|
200
|
+
const input = Object.hasOwn(event, "input") ? event.input : undefined;
|
|
201
|
+
if (input !== undefined)
|
|
202
|
+
entry.call.input = input;
|
|
203
|
+
if (!failed) {
|
|
204
|
+
const output = readToolOutput(event);
|
|
205
|
+
if (output !== undefined)
|
|
206
|
+
entry.call.output = output;
|
|
207
|
+
}
|
|
208
|
+
entry.call.status = failed
|
|
209
|
+
? isDeniedToolResult(event, error) ? "denied" : "error"
|
|
210
|
+
: entry.call.status ?? "ok";
|
|
211
|
+
if (error)
|
|
212
|
+
entry.call.error = error;
|
|
143
213
|
}
|
|
144
214
|
}
|
|
145
|
-
return [...toolCalls.values()];
|
|
215
|
+
return [...toolCalls.values()].map((entry) => entry.call);
|
|
146
216
|
}
|
|
147
217
|
function createRunOutput(run) {
|
|
148
218
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"expect.d.ts","sourceRoot":"","sources":["../../../src/src/eval/expect.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"expect.d.ts","sourceRoot":"","sources":["../../../src/src/eval/expect.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EACV,cAAc,EACd,WAAW,EACX,UAAU,EAEV,gBAAgB,EAEhB,UAAU,EAEX,MAAM,YAAY,CAAC;AAuBpB,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,gBAAgB,EAAE,GAAG,UAAU,CA6D3F;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE;IAC5C,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,MAAM,EAAE,gBAAgB,EAAE,CAAC;CAC5B;;;;;;EAQA"}
|
package/esm/src/eval/expect.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { getOutputText } from "./metrics.js";
|
|
2
|
+
import { evaluateCalledTool, evaluateNotCalledTool, evaluateToolCallCount, isEvalToolFailed, } from "./tool-behavior.js";
|
|
2
3
|
function createExpectation(checks, base) {
|
|
3
4
|
function record(severity, threshold) {
|
|
4
5
|
const check = {
|
|
@@ -39,7 +40,7 @@ export function createEvalExpect(record, checks) {
|
|
|
39
40
|
},
|
|
40
41
|
noFailedTools() {
|
|
41
42
|
const failedTools = record.trace.toolCalls
|
|
42
|
-
.filter(
|
|
43
|
+
.filter(isEvalToolFailed)
|
|
43
44
|
.map((tool) => tool.name);
|
|
44
45
|
return createExpectation(checks, {
|
|
45
46
|
name: "expect.noFailedTools",
|
|
@@ -49,6 +50,27 @@ export function createEvalExpect(record, checks) {
|
|
|
49
50
|
...(failedTools.length > 0 ? { evidence: { failedTools } } : {}),
|
|
50
51
|
});
|
|
51
52
|
},
|
|
53
|
+
calledTool(name, options) {
|
|
54
|
+
return createExpectation(checks, {
|
|
55
|
+
name: "expect.calledTool",
|
|
56
|
+
family: "check",
|
|
57
|
+
...evaluateCalledTool(record, name, options),
|
|
58
|
+
});
|
|
59
|
+
},
|
|
60
|
+
notCalledTool(name) {
|
|
61
|
+
return createExpectation(checks, {
|
|
62
|
+
name: "expect.notCalledTool",
|
|
63
|
+
family: "check",
|
|
64
|
+
...evaluateNotCalledTool(record, name),
|
|
65
|
+
});
|
|
66
|
+
},
|
|
67
|
+
toolCallCount(name, options) {
|
|
68
|
+
return createExpectation(checks, {
|
|
69
|
+
name: "expect.toolCallCount",
|
|
70
|
+
family: "check",
|
|
71
|
+
...evaluateToolCallCount(record, name, options),
|
|
72
|
+
});
|
|
73
|
+
},
|
|
52
74
|
};
|
|
53
75
|
}
|
|
54
76
|
export function createEvalCheckContext(input) {
|
package/esm/src/eval/index.d.ts
CHANGED
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
* ]),
|
|
15
15
|
* metrics: [
|
|
16
16
|
* metrics.answer.contains({ text: "Paris" }).gate(),
|
|
17
|
+
* metrics.agent.calledTool("search_docs").gate(),
|
|
17
18
|
* metrics.agent.noFailedTools().gate(),
|
|
18
19
|
* ],
|
|
19
20
|
* });
|
|
@@ -51,6 +52,6 @@ export { runEval } from "./runner.js";
|
|
|
51
52
|
export { deriveEvalId, discoverEvals, findEvalById } from "./discovery.js";
|
|
52
53
|
export { createEvalSourceDocument, getEvalEditableFieldSchema, getEvalRunSchema, getEvalSourceDocumentSchema, getEvalSourcePatchSchema, getEvalSourceReferenceSchema, getEvalStudioCapabilitySchema, } from "./studio.js";
|
|
53
54
|
export type { DiscoveredEval, EvalDiscoveryOptions, EvalDiscoveryResult } from "./discovery.js";
|
|
54
|
-
export type { EvalAgentAdapter, EvalAgentAdapterContext, EvalAgentAdapterResult, EvalAgentInput, EvalCheckContext, EvalDataset, EvalDatasetLoadContext, EvalDefinition, EvalDurationSummary, EvalExample, EvalExampleInput, EvalExpect, EvalExpectation, EvalFailedExampleSummary, EvalFlakeSummary, EvalGateFailureSummary, EvalMetric, EvalMetricContext, EvalMetricDeltaSummary, EvalMetricFamily, EvalMetricResult, EvalMetricSummary, EvalMetricThreshold, EvalRecord, EvalReport, EvalReportComparison, EvalReportExportConfig, EvalReportSummary, EvalSeverity, EvalSource, EvalTargetKind, EvalToolCall, EvalTrace, EvalUsage, EvalUsageSummary, RunEvalOptions, } from "./types.js";
|
|
55
|
+
export type { EvalAgentAdapter, EvalAgentAdapterContext, EvalAgentAdapterResult, EvalAgentInput, EvalCheckContext, EvalDataset, EvalDatasetLoadContext, EvalDefinition, EvalDurationSummary, EvalExample, EvalExampleInput, EvalExpect, EvalExpectation, EvalFailedExampleSummary, EvalFlakeSummary, EvalGateFailureSummary, EvalMetric, EvalMetricContext, EvalMetricDeltaSummary, EvalMetricFamily, EvalMetricResult, EvalMetricSummary, EvalMetricThreshold, EvalRecord, EvalReport, EvalReportComparison, EvalReportExportConfig, EvalReportSummary, EvalSeverity, EvalSource, EvalTargetKind, EvalToolCall, EvalToolCallCountOptions, EvalToolCallMatchOptions, EvalToolCallStatus, EvalToolInputMatchMode, EvalTrace, EvalUsage, EvalUsageSummary, RunEvalOptions, } from "./types.js";
|
|
55
56
|
export type { CreateEvalSourceDocumentOptions, EvalEditableField, EvalRun, EvalSourceDocument, EvalSourcePatch, EvalSourceReference, EvalStudioCapability, } from "./studio.js";
|
|
56
57
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/eval/index.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AACH,OAAO,yBAAyB,CAAC;AAGjC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC3E,OAAO,EACL,wBAAwB,EACxB,0BAA0B,EAC1B,gBAAgB,EAChB,2BAA2B,EAC3B,wBAAwB,EACxB,4BAA4B,EAC5B,6BAA6B,GAC9B,MAAM,aAAa,CAAC;AAErB,YAAY,EAAE,cAAc,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAChG,YAAY,EACV,gBAAgB,EAChB,uBAAuB,EACvB,sBAAsB,EACtB,cAAc,EACd,gBAAgB,EAChB,WAAW,EACX,sBAAsB,EACtB,cAAc,EACd,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,UAAU,EACV,eAAe,EACf,wBAAwB,EACxB,gBAAgB,EAChB,sBAAsB,EACtB,UAAU,EACV,iBAAiB,EACjB,sBAAsB,EACtB,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,EACnB,UAAU,EACV,UAAU,EACV,oBAAoB,EACpB,sBAAsB,EACtB,iBAAiB,EACjB,YAAY,EACZ,UAAU,EACV,cAAc,EACd,YAAY,EACZ,wBAAwB,EACxB,wBAAwB,EACxB,kBAAkB,EAClB,sBAAsB,EACtB,SAAS,EACT,SAAS,EACT,gBAAgB,EAChB,cAAc,GACf,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,+BAA+B,EAC/B,iBAAiB,EACjB,OAAO,EACP,kBAAkB,EAClB,eAAe,EACf,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,aAAa,CAAC"}
|
package/esm/src/eval/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { EvalMetric } from "./types.js";
|
|
1
|
+
import type { EvalMetric, EvalToolCallCountOptions, EvalToolCallMatchOptions } from "./types.js";
|
|
2
2
|
type JudgeRubricInput = {
|
|
3
3
|
rubric: string;
|
|
4
4
|
judge?: (input: {
|
|
@@ -33,6 +33,9 @@ export declare const metrics: {
|
|
|
33
33
|
};
|
|
34
34
|
readonly agent: {
|
|
35
35
|
readonly noFailedTools: () => EvalMetric;
|
|
36
|
+
readonly calledTool: (name: string, options?: EvalToolCallMatchOptions) => EvalMetric;
|
|
37
|
+
readonly notCalledTool: (name: string) => EvalMetric;
|
|
38
|
+
readonly toolCallCount: (name: string, options: EvalToolCallCountOptions) => EvalMetric;
|
|
36
39
|
};
|
|
37
40
|
readonly ops: {
|
|
38
41
|
readonly latency: (options: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../../src/src/eval/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,UAAU,
|
|
1
|
+
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../../src/src/eval/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,UAAU,EAMV,wBAAwB,EACxB,wBAAwB,EACzB,MAAM,YAAY,CAAC;AAUpB,KAAK,gBAAgB,GAAG;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE;QACd,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,OAAO,CAAC;QACf,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAChC,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACnC,KAAK,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACxE,CAAC;AAEF,iBAAS,aAAa,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,CAQ9C;AASD,iBAAS,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAE/C;AAuFD,0FAA0F;AAC1F,eAAO,MAAM,OAAO;;mCAEF,UAAU;qCAON;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,aAAa,CAAC,EAAE,OAAO,CAAA;SAAE,KAAG,UAAU;kCAWzD;YAAE,OAAO,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAA;SAAE,KAAG,UAAU;sCAY5C;YAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;SAAE,KAAG,UAAU;;;sCAWrC,UAAU;oCAgBV,MAAM,YAAY,wBAAwB,KAAG,UAAU;uCAcpD,MAAM,KAAG,UAAU;uCAcnB,MAAM,WAAW,wBAAwB,KAAG,UAAU;;;oCAgBzD;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,KAAG,UAAU;mCAc/B;YAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;YAAC,SAAS,CAAC,EAAE,MAAM,CAAA;SAAE,KAAG,UAAU;iCAoB3E;YAAE,MAAM,EAAE,MAAM,CAAA;SAAE,KAAG,UAAU;;;mCAiB7B,gBAAgB,KAAG,UAAU;;CAmCvC,CAAC;AAEX,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,CAAC"}
|
package/esm/src/eval/metrics.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { evaluateCalledTool, evaluateNotCalledTool, evaluateToolCallCount, isEvalToolFailed, } from "./tool-behavior.js";
|
|
1
2
|
function getOutputText(output) {
|
|
2
3
|
if (typeof output === "string")
|
|
3
4
|
return output;
|
|
@@ -85,9 +86,6 @@ function createMetric(name, family, evaluator, config) {
|
|
|
85
86
|
function scoreResult(name, family, severity, pass, score = pass ? 1 : 0) {
|
|
86
87
|
return { name, family, severity, score, pass };
|
|
87
88
|
}
|
|
88
|
-
function isToolFailed(toolCall) {
|
|
89
|
-
return toolCall.status === "error" || typeof toolCall.error === "string";
|
|
90
|
-
}
|
|
91
89
|
/** Metric factories for deterministic answers, agent behavior, operations, and judges. */
|
|
92
90
|
export const metrics = {
|
|
93
91
|
answer: {
|
|
@@ -125,7 +123,7 @@ export const metrics = {
|
|
|
125
123
|
agent: {
|
|
126
124
|
noFailedTools() {
|
|
127
125
|
return createMetric("agent.noFailedTools", "agent", (record) => {
|
|
128
|
-
const failedTools = record.trace.toolCalls.filter(
|
|
126
|
+
const failedTools = record.trace.toolCalls.filter(isEvalToolFailed).map((tool) => tool.name);
|
|
129
127
|
return {
|
|
130
128
|
name: "agent.noFailedTools",
|
|
131
129
|
family: "agent",
|
|
@@ -136,6 +134,30 @@ export const metrics = {
|
|
|
136
134
|
};
|
|
137
135
|
});
|
|
138
136
|
},
|
|
137
|
+
calledTool(name, options) {
|
|
138
|
+
return createMetric("agent.calledTool", "agent", (record) => ({
|
|
139
|
+
name: "agent.calledTool",
|
|
140
|
+
family: "agent",
|
|
141
|
+
severity: "gate",
|
|
142
|
+
...evaluateCalledTool(record, name, options),
|
|
143
|
+
}), { tool: name, ...(options ?? {}) });
|
|
144
|
+
},
|
|
145
|
+
notCalledTool(name) {
|
|
146
|
+
return createMetric("agent.notCalledTool", "agent", (record) => ({
|
|
147
|
+
name: "agent.notCalledTool",
|
|
148
|
+
family: "agent",
|
|
149
|
+
severity: "gate",
|
|
150
|
+
...evaluateNotCalledTool(record, name),
|
|
151
|
+
}), { tool: name });
|
|
152
|
+
},
|
|
153
|
+
toolCallCount(name, options) {
|
|
154
|
+
return createMetric("agent.toolCallCount", "agent", (record) => ({
|
|
155
|
+
name: "agent.toolCallCount",
|
|
156
|
+
family: "agent",
|
|
157
|
+
severity: "gate",
|
|
158
|
+
...evaluateToolCallCount(record, name, options),
|
|
159
|
+
}), { tool: name, ...options });
|
|
160
|
+
},
|
|
139
161
|
},
|
|
140
162
|
ops: {
|
|
141
163
|
latency(options) {
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { EvalMetricResult, EvalRecord, EvalToolCall, EvalToolCallCountOptions, EvalToolCallMatchOptions } from "./types.js";
|
|
2
|
+
type ToolBehaviorResult = Omit<EvalMetricResult, "name" | "family" | "severity">;
|
|
3
|
+
export declare function findEvalToolCalls(record: EvalRecord, name: string): EvalToolCall[];
|
|
4
|
+
export declare function isEvalToolFailed(toolCall: {
|
|
5
|
+
status?: string;
|
|
6
|
+
error?: string;
|
|
7
|
+
}): boolean;
|
|
8
|
+
export declare function evaluateCalledTool(record: EvalRecord, name: string, options?: EvalToolCallMatchOptions): ToolBehaviorResult;
|
|
9
|
+
export declare function evaluateNotCalledTool(record: EvalRecord, name: string): ToolBehaviorResult;
|
|
10
|
+
export declare function evaluateToolCallCount(record: EvalRecord, name: string, options: EvalToolCallCountOptions): ToolBehaviorResult;
|
|
11
|
+
export {};
|
|
12
|
+
//# sourceMappingURL=tool-behavior.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tool-behavior.d.ts","sourceRoot":"","sources":["../../../src/src/eval/tool-behavior.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,UAAU,EACV,YAAY,EACZ,wBAAwB,EACxB,wBAAwB,EAEzB,MAAM,YAAY,CAAC;AAEpB,KAAK,kBAAkB,GAAG,IAAI,CAAC,gBAAgB,EAAE,MAAM,GAAG,QAAQ,GAAG,UAAU,CAAC,CAAC;AAkEjF,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,YAAY,EAAE,CAElF;AAED,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAGvF;AAED,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,UAAU,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,wBAA6B,GACrC,kBAAkB,CAuBpB;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,kBAAkB,CAQ1F;AAED,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,UAAU,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,wBAAwB,GAChC,kBAAkB,CAYpB"}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
function sortJsonValue(value) {
|
|
2
|
+
if (Array.isArray(value))
|
|
3
|
+
return value.map(sortJsonValue);
|
|
4
|
+
if (!value || typeof value !== "object")
|
|
5
|
+
return value;
|
|
6
|
+
const sorted = {};
|
|
7
|
+
for (const key of Object.keys(value).sort()) {
|
|
8
|
+
sorted[key] = sortJsonValue(value[key]);
|
|
9
|
+
}
|
|
10
|
+
return sorted;
|
|
11
|
+
}
|
|
12
|
+
function stableStringify(value) {
|
|
13
|
+
return JSON.stringify(sortJsonValue(value));
|
|
14
|
+
}
|
|
15
|
+
function valuesEqual(actual, expected) {
|
|
16
|
+
return stableStringify(actual) === stableStringify(expected);
|
|
17
|
+
}
|
|
18
|
+
function isRecord(value) {
|
|
19
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
20
|
+
}
|
|
21
|
+
function partialMatch(actual, expected) {
|
|
22
|
+
if (Array.isArray(expected)) {
|
|
23
|
+
return Array.isArray(actual) &&
|
|
24
|
+
expected.every((entry, index) => partialMatch(actual[index], entry));
|
|
25
|
+
}
|
|
26
|
+
if (isRecord(expected)) {
|
|
27
|
+
if (!isRecord(actual))
|
|
28
|
+
return false;
|
|
29
|
+
return Object.entries(expected).every(([key, value]) => partialMatch(actual[key], value));
|
|
30
|
+
}
|
|
31
|
+
return valuesEqual(actual, expected);
|
|
32
|
+
}
|
|
33
|
+
function matchesInput(actual, expected, mode) {
|
|
34
|
+
return mode === "exact" ? valuesEqual(actual, expected) : partialMatch(actual, expected);
|
|
35
|
+
}
|
|
36
|
+
function hasExpectedInput(options) {
|
|
37
|
+
return Object.hasOwn(options, "input");
|
|
38
|
+
}
|
|
39
|
+
function expectedCount(options) {
|
|
40
|
+
return {
|
|
41
|
+
...(options.exact !== undefined ? { exact: options.exact } : {}),
|
|
42
|
+
...(options.min !== undefined ? { min: options.min } : {}),
|
|
43
|
+
...(options.max !== undefined ? { max: options.max } : {}),
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
function countPasses(count, options) {
|
|
47
|
+
if (options.exact !== undefined)
|
|
48
|
+
return count === options.exact;
|
|
49
|
+
const minPass = options.min === undefined || count >= options.min;
|
|
50
|
+
const maxPass = options.max === undefined || count <= options.max;
|
|
51
|
+
return minPass && maxPass;
|
|
52
|
+
}
|
|
53
|
+
export function findEvalToolCalls(record, name) {
|
|
54
|
+
return record.trace.toolCalls.filter((tool) => tool.name === name);
|
|
55
|
+
}
|
|
56
|
+
export function isEvalToolFailed(toolCall) {
|
|
57
|
+
return toolCall.status === "error" || toolCall.status === "denied" ||
|
|
58
|
+
typeof toolCall.error === "string";
|
|
59
|
+
}
|
|
60
|
+
export function evaluateCalledTool(record, name, options = {}) {
|
|
61
|
+
const calls = findEvalToolCalls(record, name);
|
|
62
|
+
const expectedInputProvided = hasExpectedInput(options);
|
|
63
|
+
const match = options.match ?? "partial";
|
|
64
|
+
const inputMatched = !expectedInputProvided ||
|
|
65
|
+
calls.some((call) => matchesInput(call.input, options.input, match));
|
|
66
|
+
const pass = calls.length > 0 && inputMatched;
|
|
67
|
+
const evidence = {
|
|
68
|
+
tool: name,
|
|
69
|
+
calls: calls.length,
|
|
70
|
+
};
|
|
71
|
+
if (expectedInputProvided) {
|
|
72
|
+
evidence.expectedInput = options.input;
|
|
73
|
+
evidence.match = match;
|
|
74
|
+
if (!inputMatched)
|
|
75
|
+
evidence.actualInputs = calls.map((call) => call.input);
|
|
76
|
+
}
|
|
77
|
+
return {
|
|
78
|
+
score: pass ? 1 : 0,
|
|
79
|
+
pass,
|
|
80
|
+
evidence,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
export function evaluateNotCalledTool(record, name) {
|
|
84
|
+
const calls = findEvalToolCalls(record, name);
|
|
85
|
+
const pass = calls.length === 0;
|
|
86
|
+
return {
|
|
87
|
+
score: pass ? 1 : 0,
|
|
88
|
+
pass,
|
|
89
|
+
evidence: { tool: name, calls: calls.length },
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
export function evaluateToolCallCount(record, name, options) {
|
|
93
|
+
const calls = findEvalToolCalls(record, name);
|
|
94
|
+
const pass = countPasses(calls.length, options);
|
|
95
|
+
return {
|
|
96
|
+
score: pass ? 1 : 0,
|
|
97
|
+
pass,
|
|
98
|
+
evidence: {
|
|
99
|
+
tool: name,
|
|
100
|
+
calls: calls.length,
|
|
101
|
+
expected: expectedCount(options),
|
|
102
|
+
},
|
|
103
|
+
};
|
|
104
|
+
}
|
package/esm/src/eval/types.d.ts
CHANGED
|
@@ -49,10 +49,28 @@ export interface EvalUsage {
|
|
|
49
49
|
totalTokens?: number;
|
|
50
50
|
costUsd?: number;
|
|
51
51
|
}
|
|
52
|
+
/** Normalized status for a tool call captured during an eval record. */
|
|
53
|
+
export type EvalToolCallStatus = "ok" | "error" | "skipped" | "denied";
|
|
54
|
+
/** How expected tool input is compared to the captured tool input. */
|
|
55
|
+
export type EvalToolInputMatchMode = "exact" | "partial";
|
|
56
|
+
/** Options for matching a required tool call. */
|
|
57
|
+
export interface EvalToolCallMatchOptions {
|
|
58
|
+
input?: unknown;
|
|
59
|
+
match?: EvalToolInputMatchMode;
|
|
60
|
+
}
|
|
61
|
+
/** Options for checking how often a tool was called. */
|
|
62
|
+
export interface EvalToolCallCountOptions {
|
|
63
|
+
exact?: number;
|
|
64
|
+
min?: number;
|
|
65
|
+
max?: number;
|
|
66
|
+
}
|
|
52
67
|
/** Tool call metadata captured during one eval record. */
|
|
53
68
|
export interface EvalToolCall {
|
|
69
|
+
id?: string;
|
|
54
70
|
name: string;
|
|
55
|
-
status?:
|
|
71
|
+
status?: EvalToolCallStatus;
|
|
72
|
+
input?: unknown;
|
|
73
|
+
output?: unknown;
|
|
56
74
|
error?: string;
|
|
57
75
|
metadata?: Record<string, unknown>;
|
|
58
76
|
}
|
|
@@ -118,6 +136,9 @@ export interface EvalExpect {
|
|
|
118
136
|
completed(): EvalExpectation;
|
|
119
137
|
outputContains(text: string): EvalExpectation;
|
|
120
138
|
noFailedTools(): EvalExpectation;
|
|
139
|
+
calledTool(name: string, options?: EvalToolCallMatchOptions): EvalExpectation;
|
|
140
|
+
notCalledTool(name: string): EvalExpectation;
|
|
141
|
+
toolCallCount(name: string, options: EvalToolCallCountOptions): EvalExpectation;
|
|
121
142
|
}
|
|
122
143
|
/** Context passed to an eval definition's `check` callback. */
|
|
123
144
|
export interface EvalCheckContext {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/src/eval/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EACV,uBAAuB,EACvB,0BAA0B,EAC1B,sBAAsB,EACvB,MAAM,6BAA6B,CAAC;AAErC,qEAAqE;AACrE,MAAM,MAAM,cAAc,GAAG,OAAO,CAAC;AAErC,yDAAyD;AACzD,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAC;AAEtD,wDAAwD;AACxD,MAAM,MAAM,gBAAgB,GAAG,QAAQ,GAAG,OAAO,GAAG,KAAK,GAAG,OAAO,GAAG,OAAO,CAAC;AAE9E,yDAAyD;AACzD,MAAM,MAAM,mBAAmB,GAAG;IAChC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,gEAAgE;AAChE,MAAM,MAAM,gBAAgB,CAAC,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;AAEjD,mEAAmE;AACnE,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,OAAO,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,0DAA0D;AAC1D,MAAM,MAAM,gBAAgB,GAAG;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,OAAO,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,CAAC;AAEF,yCAAyC;AACzC,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,iDAAiD;AACjD,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,OAAO,CAAC;IAClC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,WAAW,EAAE,CAAC;IACzB,IAAI,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;CAC/D;AAED,yDAAyD;AACzD,MAAM,WAAW,SAAS;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,0DAA0D;AAC1D,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/src/eval/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EACV,uBAAuB,EACvB,0BAA0B,EAC1B,sBAAsB,EACvB,MAAM,6BAA6B,CAAC;AAErC,qEAAqE;AACrE,MAAM,MAAM,cAAc,GAAG,OAAO,CAAC;AAErC,yDAAyD;AACzD,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAC;AAEtD,wDAAwD;AACxD,MAAM,MAAM,gBAAgB,GAAG,QAAQ,GAAG,OAAO,GAAG,KAAK,GAAG,OAAO,GAAG,OAAO,CAAC;AAE9E,yDAAyD;AACzD,MAAM,MAAM,mBAAmB,GAAG;IAChC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,gEAAgE;AAChE,MAAM,MAAM,gBAAgB,CAAC,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;AAEjD,mEAAmE;AACnE,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,OAAO,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,0DAA0D;AAC1D,MAAM,MAAM,gBAAgB,GAAG;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,OAAO,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,CAAC;AAEF,yCAAyC;AACzC,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,iDAAiD;AACjD,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,OAAO,CAAC;IAClC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,WAAW,EAAE,CAAC;IACzB,IAAI,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;CAC/D;AAED,yDAAyD;AACzD,MAAM,WAAW,SAAS;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,wEAAwE;AACxE,MAAM,MAAM,kBAAkB,GAAG,IAAI,GAAG,OAAO,GAAG,SAAS,GAAG,QAAQ,CAAC;AAEvE,sEAAsE;AACtE,MAAM,MAAM,sBAAsB,GAAG,OAAO,GAAG,SAAS,CAAC;AAEzD,iDAAiD;AACjD,MAAM,WAAW,wBAAwB;IACvC,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,sBAAsB,CAAC;CAChC;AAED,wDAAwD;AACxD,MAAM,WAAW,wBAAwB;IACvC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,0DAA0D;AAC1D,MAAM,WAAW,YAAY;IAC3B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,mDAAmD;AACnD,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,OAAO,EAAE,CAAC;IAClB,SAAS,EAAE,YAAY,EAAE,CAAC;CAC3B;AAED,iEAAiE;AACjE,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC7B,MAAM,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC7B;AAED,qDAAqD;AACrD,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,4DAA4D;AAC5D,MAAM,WAAW,iBAAiB;IAChC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAClB;AAED,gDAAgD;AAChD,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,SAAS,CAAC,EAAE,mBAAmB,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,gBAAgB,CAAC,gBAAgB,CAAC,CAAC;IAC9F,IAAI,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,UAAU,CAAC;IAClD,IAAI,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,UAAU,CAAC;IAClD,MAAM,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,UAAU,CAAC;CACrD;AAED,wDAAwD;AACxD,MAAM,WAAW,eAAe;IAC9B,IAAI,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,gBAAgB,CAAC;IACxD,IAAI,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,gBAAgB,CAAC;IACxD,MAAM,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,gBAAgB,CAAC;CAC3D;AAED,6DAA6D;AAC7D,MAAM,WAAW,UAAU;IACzB,SAAS,IAAI,eAAe,CAAC;IAC7B,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,CAAC;IAC9C,aAAa,IAAI,eAAe,CAAC;IACjC,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,wBAAwB,GAAG,eAAe,CAAC;IAC9E,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,CAAC;IAC7C,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,wBAAwB,GAAG,eAAe,CAAC;CACjF;AAED,+DAA+D;AAC/D,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;CACpB;AAED,wDAAwD;AACxD,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,kEAAkE;AAClE,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,WAAW,CAAC;IACrB,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,gBAAgB,KAAK,gBAAgB,CAAC,IAAI,CAAC,CAAC;CAC/D;AAED,qCAAqC;AACrC,MAAM,WAAW,cAAc;IAC7B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,WAAW,GAAG,gBAAgB,EAAE,CAAC;IAC1C,OAAO,CAAC,EAAE,UAAU,EAAE,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,gBAAgB,KAAK,gBAAgB,CAAC,IAAI,CAAC,CAAC;CAC/D;AAED,6EAA6E;AAC7E,MAAM,WAAW,uBAAuB;IACtC,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,2DAA2D;AAC3D,MAAM,WAAW,sBAAsB;IACrC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAC3B,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,6DAA6D;AAC7D,MAAM,MAAM,gBAAgB,GAAG,CAC7B,OAAO,EAAE,uBAAuB,KAC7B,gBAAgB,CAAC,MAAM,GAAG,sBAAsB,CAAC,CAAC;AAEvD,2CAA2C;AAC3C,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE;QACR,KAAK,EAAE,gBAAgB,CAAC;KACzB,CAAC;IACF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IACjB,MAAM,CAAC,EAAE,sBAAsB,CAAC;CACjC;AAED,wDAAwD;AACxD,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,EAAE,0BAA0B,CAAC;IACtC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,OAAO,CAAC,EAAE,uBAAuB,CAAC;CACnC;AAED,kDAAkD;AAClD,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,6CAA6C;AAC7C,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED,uCAAuC;AACvC,MAAM,WAAW,gBAAgB;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qDAAqD;AACrD,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,kEAAkE;AAClE,MAAM,WAAW,wBAAwB;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,OAAO,CAAC;CAChB;AAED,uDAAuD;AACvD,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,4EAA4E;AAC5E,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,qDAAqD;AACrD,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,wBAAwB,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,sBAAsB,EAAE,CAAC;IACvC,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,uDAAuD;AACvD,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,iDAAiD;IACjD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,yCAAyC;IACzC,QAAQ,CAAC,EAAE,mBAAmB,CAAC;IAC/B,mCAAmC;IACnC,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,uEAAuE;IACvE,YAAY,CAAC,EAAE,sBAAsB,EAAE,CAAC;IACxC,oEAAoE;IACpE,cAAc,CAAC,EAAE,wBAAwB,EAAE,CAAC;IAC5C,6CAA6C;IAC7C,MAAM,CAAC,EAAE,gBAAgB,CAAC;CAC3B;AAED,sDAAsD;AACtD,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,cAAc,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,OAAO,CAAC,EAAE,sBAAsB,EAAE,CAAC;CACpC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agent-stream.handler.d.ts","sourceRoot":"","sources":["../../../../../src/src/server/handlers/request/agent-stream.handler.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,KAAK,yBAAyB,EAAE,MAAM,oCAAoC,CAAC;AACpF,OAAO,EAEL,KAAK,+BAA+B,EACrC,MAAM,wCAAwC,CAAC;AAOhD,OAAO,EACL,4BAA4B,EAE7B,MAAM,2CAA2C,CAAC;
|
|
1
|
+
{"version":3,"file":"agent-stream.handler.d.ts","sourceRoot":"","sources":["../../../../../src/src/server/handlers/request/agent-stream.handler.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,KAAK,yBAAyB,EAAE,MAAM,oCAAoC,CAAC;AACpF,OAAO,EAEL,KAAK,+BAA+B,EACrC,MAAM,wCAAwC,CAAC;AAOhD,OAAO,EACL,4BAA4B,EAE7B,MAAM,2CAA2C,CAAC;AAyBnD,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAmB,aAAa,EAAE,MAAM,aAAa,CAAC;AAanG,MAAM,WAAW,sBACf,SAAQ,yBAAyB,EAAE,+BAA+B;IAClE,4BAA4B,CAAC,EAAE,OAAO,4BAA4B,CAAC;CACpE;AAqdD,qBAAa,kBAAmB,SAAQ,WAAW;IASrC,OAAO,CAAC,QAAQ,CAAC,IAAI;IARjC,QAAQ,EAAE,eAAe,CAMvB;gBAE2B,IAAI,GAAE,sBAAoC;IAIvE,OAAO,CAAC,sBAAsB;IAwBxB,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;CAoLxE"}
|
|
@@ -108,6 +108,58 @@ function getRequestedStudioToolNames(input) {
|
|
|
108
108
|
.filter((toolName) => STUDIO_RUNTIME_REMOTE_TOOL_NAMES.has(toolName))
|
|
109
109
|
.sort();
|
|
110
110
|
}
|
|
111
|
+
function sanitizeForwardedRuntimeAllowedTools(input) {
|
|
112
|
+
const forwardedProps = input.forwardedProps;
|
|
113
|
+
if (!isRecord(forwardedProps)) {
|
|
114
|
+
return forwardedProps;
|
|
115
|
+
}
|
|
116
|
+
const runtimeOverrides = isRecord(forwardedProps.runtimeOverrides)
|
|
117
|
+
? forwardedProps.runtimeOverrides
|
|
118
|
+
: null;
|
|
119
|
+
if (!runtimeOverrides || !Object.hasOwn(runtimeOverrides, "allowedTools")) {
|
|
120
|
+
return forwardedProps;
|
|
121
|
+
}
|
|
122
|
+
const allowedTools = runtimeOverrides.allowedTools;
|
|
123
|
+
if (!Array.isArray(allowedTools) || !allowedTools.every((toolName) => typeof toolName === "string")) {
|
|
124
|
+
return forwardedProps;
|
|
125
|
+
}
|
|
126
|
+
const availableToolNames = new Set(input.availableToolNames);
|
|
127
|
+
// Platform remote tools are gated separately by the child agent config in
|
|
128
|
+
// withVeryfrontPlatformRemoteTools. The Studio path is the one that consumes
|
|
129
|
+
// forwarded allowedTools, and Studio-only runtime tools are preserved only
|
|
130
|
+
// for trusted Studio clients that can already attach the Studio MCP surface.
|
|
131
|
+
const sanitizedAllowedTools = allowedTools.filter((toolName) => availableToolNames.has(toolName) ||
|
|
132
|
+
(input.allowStudioRuntimeTools && STUDIO_RUNTIME_REMOTE_TOOL_NAMES.has(toolName)));
|
|
133
|
+
if (sanitizedAllowedTools.length === allowedTools.length) {
|
|
134
|
+
return forwardedProps;
|
|
135
|
+
}
|
|
136
|
+
const nextRuntimeOverrides = {
|
|
137
|
+
...runtimeOverrides,
|
|
138
|
+
allowedTools: sanitizedAllowedTools,
|
|
139
|
+
};
|
|
140
|
+
if (sanitizedAllowedTools.length === 0) {
|
|
141
|
+
delete nextRuntimeOverrides.allowedTools;
|
|
142
|
+
}
|
|
143
|
+
const nextForwardedProps = {
|
|
144
|
+
...forwardedProps,
|
|
145
|
+
runtimeOverrides: nextRuntimeOverrides,
|
|
146
|
+
};
|
|
147
|
+
if (Object.keys(nextRuntimeOverrides).length === 0) {
|
|
148
|
+
delete nextForwardedProps.runtimeOverrides;
|
|
149
|
+
}
|
|
150
|
+
return Object.keys(nextForwardedProps).length > 0 ? nextForwardedProps : undefined;
|
|
151
|
+
}
|
|
152
|
+
function sanitizeRuntimeRunAgentInput(input) {
|
|
153
|
+
const clientProfile = resolveRuntimeClientProfile(input.forwardedProps);
|
|
154
|
+
return {
|
|
155
|
+
...input,
|
|
156
|
+
forwardedProps: sanitizeForwardedRuntimeAllowedTools({
|
|
157
|
+
forwardedProps: input.forwardedProps,
|
|
158
|
+
availableToolNames: input.tools.map((tool) => tool.name),
|
|
159
|
+
allowStudioRuntimeTools: clientAllowsStudioMcp(clientProfile),
|
|
160
|
+
}),
|
|
161
|
+
};
|
|
162
|
+
}
|
|
111
163
|
function getVeryfrontApiMcpPolicy(agent) {
|
|
112
164
|
const requestedToolNames = new Set();
|
|
113
165
|
const deniedToolNames = new Set();
|
|
@@ -368,7 +420,7 @@ export class AgentStreamHandler extends BaseHandler {
|
|
|
368
420
|
});
|
|
369
421
|
return this.respond(builder.json({ error: "Agent not found" }, 404));
|
|
370
422
|
}
|
|
371
|
-
const runtimeInput = toRuntimeRunAgentInput(payload);
|
|
423
|
+
const runtimeInput = sanitizeRuntimeRunAgentInput(toRuntimeRunAgentInput(payload));
|
|
372
424
|
const apiAuthToken = payload.credentials?.authToken || ctx.proxyToken ||
|
|
373
425
|
getHostEnv("VERYFRONT_API_TOKEN") || "";
|
|
374
426
|
const platformRuntimeAgent = await withVeryfrontPlatformRemoteTools({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dynamic-run-entrypoint.d.ts","sourceRoot":"","sources":["../../../../src/src/workflow/worker/dynamic-run-entrypoint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;
|
|
1
|
+
{"version":3,"file":"dynamic-run-entrypoint.d.ts","sourceRoot":"","sources":["../../../../src/src/workflow/worker/dynamic-run-entrypoint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AASH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAY5D;;GAEG;AACH,eAAO,MAAM,kBAAkB;;;;;;CAMrB,CAAC;AAEX;;GAEG;AACH,MAAM,WAAW,kCAAkC;IACjD,uCAAuC;IACvC,OAAO,EAAE,eAAe,CAAC;IAEzB,2BAA2B;IAC3B,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,qBAAqB,CACzC,MAAM,EAAE,kCAAkC,GACzC,OAAO,CAAC,MAAM,CAAC,CAmIjB;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,WAAW,yCAAyC;IACxD,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAC;IAEjB,2BAA2B;IAC3B,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,gDAAgD;AAChD,wBAAsB,kCAAkC,CACtD,OAAO,EAAE,yCAAyC,GACjD,OAAO,CAAC,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,CAchC"}
|
|
@@ -28,7 +28,9 @@ import { logger as baseLogger } from "../../utils/index.js";
|
|
|
28
28
|
import { getEnv } from "../../platform/compat/process.js";
|
|
29
29
|
import { enhanceAdapterWithFS } from "../../platform/adapters/fs/integration.js";
|
|
30
30
|
import { denoAdapter } from "../../platform/adapters/runtime/deno/index.js";
|
|
31
|
-
import {
|
|
31
|
+
import { agentRegistry } from "../../agent/composition/index.js";
|
|
32
|
+
import { discoverProjectAgentRuntime } from "../../agent/project/agent-runtime.js";
|
|
33
|
+
import { toolRegistry } from "../../tool/registry.js";
|
|
32
34
|
import { WorkflowExecutor } from "../executor/workflow-executor.js";
|
|
33
35
|
import { failRunExecution, getFinalRunExitCode, getTenantFromEnv, hydrateRunContextEnv, runWithTenantContext, } from "./shared.js";
|
|
34
36
|
const logger = baseLogger.component("dynamic-workflow-run-entrypoint");
|
|
@@ -100,37 +102,41 @@ export async function runDynamicWorkflowRun(config) {
|
|
|
100
102
|
if (debug) {
|
|
101
103
|
logger.info("FS adapter initialized");
|
|
102
104
|
}
|
|
103
|
-
// Discover workflows
|
|
104
|
-
const discoveryResult = await
|
|
105
|
+
// Discover workflows and the project-local agent/tool registries they may reference.
|
|
106
|
+
const discoveryResult = await discoverProjectAgentRuntime({
|
|
105
107
|
projectDir: "", // Root of project (relative paths with API)
|
|
106
108
|
adapter,
|
|
107
|
-
|
|
108
|
-
debug,
|
|
109
|
+
verbose: debug,
|
|
109
110
|
});
|
|
110
111
|
if (discoveryResult.errors.length > 0 && debug) {
|
|
111
112
|
logger.warn("Some workflow files failed to load:", discoveryResult.errors);
|
|
112
113
|
}
|
|
113
|
-
|
|
114
|
+
const workflows = [...discoveryResult.workflows.values()];
|
|
115
|
+
if (workflows.length === 0) {
|
|
114
116
|
logger.error("No workflows discovered");
|
|
115
117
|
return DYNAMIC_EXIT_CODES.DISCOVERY_FAILED;
|
|
116
118
|
}
|
|
117
119
|
if (debug) {
|
|
118
|
-
logger.info(`[DynamicWorkflowRun] Discovered ${
|
|
120
|
+
logger.info(`[DynamicWorkflowRun] Discovered ${workflows.length} workflows:`, workflows.map((w) => w.id));
|
|
119
121
|
}
|
|
120
122
|
// Find the matching workflow
|
|
121
|
-
const workflow =
|
|
123
|
+
const workflow = workflows.find((w) => w.id === run.workflowId);
|
|
122
124
|
if (!workflow) {
|
|
123
125
|
logger.error(`Workflow not found: ${run.workflowId}`);
|
|
124
|
-
logger.error(`[DynamicWorkflowRun] Available workflows: ${
|
|
126
|
+
logger.error(`[DynamicWorkflowRun] Available workflows: ${workflows.map((w) => w.id).join(", ")}`);
|
|
125
127
|
return DYNAMIC_EXIT_CODES.NOT_FOUND;
|
|
126
128
|
}
|
|
127
129
|
if (debug) {
|
|
128
|
-
logger.info(`Found workflow "${workflow.id}"
|
|
130
|
+
logger.info(`Found workflow "${workflow.id}"`);
|
|
129
131
|
}
|
|
130
132
|
// Create executor and register the workflow
|
|
131
133
|
const executor = new WorkflowExecutor({
|
|
132
134
|
backend,
|
|
133
135
|
debug,
|
|
136
|
+
stepExecutor: {
|
|
137
|
+
agentRegistry,
|
|
138
|
+
toolRegistry,
|
|
139
|
+
},
|
|
134
140
|
});
|
|
135
141
|
executor.register(workflow.definition);
|
|
136
142
|
// Execute the workflow
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-entrypoint.d.ts","sourceRoot":"","sources":["../../../../src/src/workflow/worker/run-entrypoint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;
|
|
1
|
+
{"version":3,"file":"run-entrypoint.d.ts","sourceRoot":"","sources":["../../../../src/src/workflow/worker/run-entrypoint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAMH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kCAAkC,CAAC;AACzE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAWtD;;GAEG;AACH,MAAM,WAAW,2BAA2B;IAC1C,uCAAuC;IACvC,OAAO,EAAE,eAAe,CAAC;IAEzB,wBAAwB;IACxB,QAAQ,EAAE,gBAAgB,CAAC;IAE3B,2BAA2B;IAC3B,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED;;GAEG;AACH,eAAO,MAAM,UAAU;;;;;CAKb,CAAC;AAEX;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,2BAA2B,GAAG,OAAO,CAAC,MAAM,CAAC,CAgEzF;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,WAAW,kCAAkC;IACjD,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAC;IAEjB,4BAA4B;IAC5B,SAAS,EAAE,KAAK,CAAC;QAAE,UAAU,EAAE,kBAAkB,CAAA;KAAE,CAAC,CAAC;IAErD,2BAA2B;IAC3B,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,wCAAwC;AACxC,wBAAsB,2BAA2B,CAC/C,OAAO,EAAE,kCAAkC,GAC1C,OAAO,CAAC,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,CA8BhC"}
|
|
@@ -21,6 +21,8 @@
|
|
|
21
21
|
*/
|
|
22
22
|
import { logger as baseLogger } from "../../utils/index.js";
|
|
23
23
|
import { getEnv } from "../../platform/compat/process.js";
|
|
24
|
+
import { agentRegistry } from "../../agent/composition/index.js";
|
|
25
|
+
import { toolRegistry } from "../../tool/registry.js";
|
|
24
26
|
import { failRunExecution, getFinalRunExitCode, getTenantFromEnv, hydrateRunContextEnv, runWithTenantContext, } from "./shared.js";
|
|
25
27
|
const logger = baseLogger.component("workflow-run-entrypoint");
|
|
26
28
|
/**
|
|
@@ -122,6 +124,10 @@ export async function createWorkflowRunEntrypoint(options) {
|
|
|
122
124
|
const executor = new WorkflowExecutor({
|
|
123
125
|
backend,
|
|
124
126
|
debug: options.debug,
|
|
127
|
+
stepExecutor: {
|
|
128
|
+
agentRegistry,
|
|
129
|
+
toolRegistry,
|
|
130
|
+
},
|
|
125
131
|
});
|
|
126
132
|
// Register workflows
|
|
127
133
|
for (const wf of options.workflows) {
|