veryfront 0.1.923 → 0.1.924
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/deno.js +1 -1
- package/esm/src/eval/agent-service.d.ts.map +1 -1
- package/esm/src/eval/agent-service.js +86 -16
- package/esm/src/eval/expect.d.ts.map +1 -1
- package/esm/src/eval/expect.js +23 -1
- package/esm/src/eval/index.d.ts +2 -1
- package/esm/src/eval/index.d.ts.map +1 -1
- package/esm/src/eval/index.js +1 -0
- package/esm/src/eval/metrics.d.ts +4 -1
- package/esm/src/eval/metrics.d.ts.map +1 -1
- package/esm/src/eval/metrics.js +26 -4
- package/esm/src/eval/tool-behavior.d.ts +12 -0
- package/esm/src/eval/tool-behavior.d.ts.map +1 -0
- package/esm/src/eval/tool-behavior.js +104 -0
- package/esm/src/eval/types.d.ts +22 -1
- package/esm/src/eval/types.d.ts.map +1 -1
- package/esm/src/server/handlers/request/agent-stream.handler.d.ts.map +1 -1
- package/esm/src/server/handlers/request/agent-stream.handler.js +53 -1
- package/esm/src/utils/version-constant.d.ts +1 -1
- package/esm/src/utils/version-constant.js +1 -1
- package/package.json +1 -1
package/esm/deno.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agent-service.d.ts","sourceRoot":"","sources":["../../../src/src/eval/agent-service.ts"],"names":[],"mappings":"AAAA,OAAO,yBAAyB,CAAC;AAEjC,OAAO,EAEL,KAAK,uBAAuB,
|
|
1
|
+
{"version":3,"file":"agent-service.d.ts","sourceRoot":"","sources":["../../../src/src/eval/agent-service.ts"],"names":[],"mappings":"AAAA,OAAO,yBAAyB,CAAC;AAEjC,OAAO,EAEL,KAAK,uBAAuB,EAM7B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EACV,gBAAgB,EAChB,uBAAuB,EAIxB,MAAM,YAAY,CAAC;AAEpB,cAAc,qCAAqC,CAAC;AACpD,cAAc,+CAA+C,CAAC;AAE9D,gEAAgE;AAChE,eAAO,MAAM,mCAAmC,oCAAoC,CAAC;AAErF,gEAAgE;AAChE,MAAM,MAAM,gCAAgC,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC,CAAC;AAE3F,gEAAgE;AAChE,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,kEAAkE;AAClE,MAAM,WAAW,0CAA0C;IACzD,EAAE,EAAE,OAAO,CAAC;IACZ,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,mEAAmE;AACnE,MAAM,WAAW,8BAA8B;IAC7C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gBAAgB,CAAC,EAAE;QACjB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;QACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AAED,4DAA4D;AAC5D,MAAM,WAAW,qCAAqC;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,OAAO,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,4DAA4D;AAC5D,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,KAAK,EAAE,EAAE,CAAC;IACV,OAAO,EAAE,EAAE,CAAC;IACZ,cAAc,CAAC,EAAE;QACf,SAAS,EAAE,8BAA8B,CAAC;KAC3C,CAAC;IACF,QAAQ,EAAE,KAAK,CAAC;QACd,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,KAAK,CAAC;YACX,IAAI,EAAE,MAAM,CAAC;YACb,IAAI,EAAE,MAAM,CAAC;SACd,CAAC,CAAC;KACJ,CAAC,CAAC;CACJ;AAED,6DAA6D;AAC7D,MAAM,WAAW,6BAA6B;IAC5C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,GAAG,GAAG,GAAG,OAAO,EAAE,IAAI,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;IACjF,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,uBAAuB,EAAE,OAAO,EAAE,uBAAuB,KAAK,IAAI,CAAC;CAC5F;AAuSD,wEAAwE;AACxE,wBAAgB,kCAAkC,CAChD,GAAG,GAAE,gCAAqC,GACzC,2BAA2B,CAoB7B;AAED,oFAAoF;AACpF,wBAAgB,mCAAmC,CACjD,GAAG,GAAE,gCAAqC,EAC1C,cAAc,SAAiD,GAC9D,0CAA0C,CAe5C;AAED,8DAA8D;AAC9D,wBAAgB,gCAAgC,CAC9C,KAAK,EAAE,qCAAqC,GAC3C,2BAA2B,CAyB7B;AAED,mGAAmG;AACnG,wBAAgB,6BAA6B,CAC3C,MAAM,EAAE,6BAA6B,GACpC,gBAAgB,CAuElB"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import "../../_dnt.polyfills.js";
|
|
2
2
|
import * as dntShim from "../../_dnt.shims.js";
|
|
3
|
-
import { agUiSseEventTypes, getAgUiSseStringField, parseAgentServiceConfig, parseAgUiSseResponse, } from "../agent/index.js";
|
|
3
|
+
import { agUiSseEventTypes, getAgUiSseStringField, mergeToolInputDelta, parseAgentServiceConfig, parseAgUiSseResponse, } from "../agent/index.js";
|
|
4
4
|
export * from "./agent-service/live-evals/index.js";
|
|
5
5
|
export * from "./agent-service/durable-run-canaries/index.js";
|
|
6
6
|
/** Default local AG-UI endpoint used by agent-service evals. */
|
|
@@ -109,6 +109,50 @@ function stringifyError(value) {
|
|
|
109
109
|
function getToolResultError(event) {
|
|
110
110
|
return stringifyError(event.result) ?? stringifyError(event.content) ?? "Tool call failed";
|
|
111
111
|
}
|
|
112
|
+
function parseJsonString(value) {
|
|
113
|
+
try {
|
|
114
|
+
return JSON.parse(value);
|
|
115
|
+
}
|
|
116
|
+
catch {
|
|
117
|
+
return value;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
function readToolInputDelta(event) {
|
|
121
|
+
return readString(event.delta) ?? readString(event.inputTextDelta) ?? readString(event.argsDelta);
|
|
122
|
+
}
|
|
123
|
+
function readToolOutput(event) {
|
|
124
|
+
if (Object.hasOwn(event, "result"))
|
|
125
|
+
return event.result;
|
|
126
|
+
if (Object.hasOwn(event, "output"))
|
|
127
|
+
return event.output;
|
|
128
|
+
if (Object.hasOwn(event, "content")) {
|
|
129
|
+
return typeof event.content === "string" ? parseJsonString(event.content) : event.content;
|
|
130
|
+
}
|
|
131
|
+
return undefined;
|
|
132
|
+
}
|
|
133
|
+
function isDeniedToolResult(event, error) {
|
|
134
|
+
const status = getAgUiSseStringField(event, "status");
|
|
135
|
+
return status === "denied" || error?.toLowerCase().includes("denied") === true;
|
|
136
|
+
}
|
|
137
|
+
function getToolCallEntry(toolCalls, key, id, name) {
|
|
138
|
+
const existing = toolCalls.get(key);
|
|
139
|
+
if (existing) {
|
|
140
|
+
if (id && !existing.call.id)
|
|
141
|
+
existing.call.id = id;
|
|
142
|
+
if (name && existing.call.name === "tool")
|
|
143
|
+
existing.call.name = name;
|
|
144
|
+
return existing;
|
|
145
|
+
}
|
|
146
|
+
const next = {
|
|
147
|
+
call: {
|
|
148
|
+
...(id ? { id } : {}),
|
|
149
|
+
name: name ?? "tool",
|
|
150
|
+
status: "ok",
|
|
151
|
+
},
|
|
152
|
+
};
|
|
153
|
+
toolCalls.set(key, next);
|
|
154
|
+
return next;
|
|
155
|
+
}
|
|
112
156
|
function createToolCalls(events) {
|
|
113
157
|
const toolCalls = new Map();
|
|
114
158
|
for (const [index, event] of events.entries()) {
|
|
@@ -119,30 +163,56 @@ function createToolCalls(events) {
|
|
|
119
163
|
continue;
|
|
120
164
|
const id = getAgUiSseStringField(event, "toolCallId");
|
|
121
165
|
const key = id ?? `name:${name}`;
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
});
|
|
166
|
+
getToolCallEntry(toolCalls, key, id, name);
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
if (type === agUiSseEventTypes.toolCallArgs) {
|
|
170
|
+
const id = getAgUiSseStringField(event, "toolCallId");
|
|
171
|
+
const toolName = getAgUiSseStringField(event, "toolCallName");
|
|
172
|
+
const key = id ?? (toolName ? `name:${toolName}` : `args:${index}`);
|
|
173
|
+
const entry = getToolCallEntry(toolCalls, key, id, toolName);
|
|
174
|
+
const input = Object.hasOwn(event, "input") ? event.input : undefined;
|
|
175
|
+
if (input !== undefined) {
|
|
176
|
+
entry.call.input = input;
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
const delta = readToolInputDelta(event);
|
|
180
|
+
if (delta === undefined)
|
|
181
|
+
continue;
|
|
182
|
+
entry.inputText = mergeToolInputDelta(entry.inputText ?? "", delta);
|
|
183
|
+
entry.call.input = parseJsonString(entry.inputText);
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
if (type === agUiSseEventTypes.toolCallEnd) {
|
|
187
|
+
const id = getAgUiSseStringField(event, "toolCallId");
|
|
188
|
+
const toolName = getAgUiSseStringField(event, "toolCallName");
|
|
189
|
+
const key = id ?? (toolName ? `name:${toolName}` : `end:${index}`);
|
|
190
|
+
getToolCallEntry(toolCalls, key, id, toolName);
|
|
129
191
|
continue;
|
|
130
192
|
}
|
|
131
193
|
if (type === agUiSseEventTypes.toolCallResult) {
|
|
132
194
|
const id = getAgUiSseStringField(event, "toolCallId");
|
|
133
195
|
const toolName = getAgUiSseStringField(event, "toolCallName");
|
|
134
196
|
const key = id ?? (toolName ? `name:${toolName}` : `result:${index}`);
|
|
135
|
-
const
|
|
197
|
+
const entry = getToolCallEntry(toolCalls, key, id, toolName);
|
|
136
198
|
const failed = event.isError === true;
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
199
|
+
const error = failed ? getToolResultError(event) : undefined;
|
|
200
|
+
const input = Object.hasOwn(event, "input") ? event.input : undefined;
|
|
201
|
+
if (input !== undefined)
|
|
202
|
+
entry.call.input = input;
|
|
203
|
+
if (!failed) {
|
|
204
|
+
const output = readToolOutput(event);
|
|
205
|
+
if (output !== undefined)
|
|
206
|
+
entry.call.output = output;
|
|
207
|
+
}
|
|
208
|
+
entry.call.status = failed
|
|
209
|
+
? isDeniedToolResult(event, error) ? "denied" : "error"
|
|
210
|
+
: entry.call.status ?? "ok";
|
|
211
|
+
if (error)
|
|
212
|
+
entry.call.error = error;
|
|
143
213
|
}
|
|
144
214
|
}
|
|
145
|
-
return [...toolCalls.values()];
|
|
215
|
+
return [...toolCalls.values()].map((entry) => entry.call);
|
|
146
216
|
}
|
|
147
217
|
function createRunOutput(run) {
|
|
148
218
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"expect.d.ts","sourceRoot":"","sources":["../../../src/src/eval/expect.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"expect.d.ts","sourceRoot":"","sources":["../../../src/src/eval/expect.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EACV,cAAc,EACd,WAAW,EACX,UAAU,EAEV,gBAAgB,EAEhB,UAAU,EAEX,MAAM,YAAY,CAAC;AAuBpB,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,gBAAgB,EAAE,GAAG,UAAU,CA6D3F;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE;IAC5C,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,MAAM,EAAE,gBAAgB,EAAE,CAAC;CAC5B;;;;;;EAQA"}
|
package/esm/src/eval/expect.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { getOutputText } from "./metrics.js";
|
|
2
|
+
import { evaluateCalledTool, evaluateNotCalledTool, evaluateToolCallCount, isEvalToolFailed, } from "./tool-behavior.js";
|
|
2
3
|
function createExpectation(checks, base) {
|
|
3
4
|
function record(severity, threshold) {
|
|
4
5
|
const check = {
|
|
@@ -39,7 +40,7 @@ export function createEvalExpect(record, checks) {
|
|
|
39
40
|
},
|
|
40
41
|
noFailedTools() {
|
|
41
42
|
const failedTools = record.trace.toolCalls
|
|
42
|
-
.filter(
|
|
43
|
+
.filter(isEvalToolFailed)
|
|
43
44
|
.map((tool) => tool.name);
|
|
44
45
|
return createExpectation(checks, {
|
|
45
46
|
name: "expect.noFailedTools",
|
|
@@ -49,6 +50,27 @@ export function createEvalExpect(record, checks) {
|
|
|
49
50
|
...(failedTools.length > 0 ? { evidence: { failedTools } } : {}),
|
|
50
51
|
});
|
|
51
52
|
},
|
|
53
|
+
calledTool(name, options) {
|
|
54
|
+
return createExpectation(checks, {
|
|
55
|
+
name: "expect.calledTool",
|
|
56
|
+
family: "check",
|
|
57
|
+
...evaluateCalledTool(record, name, options),
|
|
58
|
+
});
|
|
59
|
+
},
|
|
60
|
+
notCalledTool(name) {
|
|
61
|
+
return createExpectation(checks, {
|
|
62
|
+
name: "expect.notCalledTool",
|
|
63
|
+
family: "check",
|
|
64
|
+
...evaluateNotCalledTool(record, name),
|
|
65
|
+
});
|
|
66
|
+
},
|
|
67
|
+
toolCallCount(name, options) {
|
|
68
|
+
return createExpectation(checks, {
|
|
69
|
+
name: "expect.toolCallCount",
|
|
70
|
+
family: "check",
|
|
71
|
+
...evaluateToolCallCount(record, name, options),
|
|
72
|
+
});
|
|
73
|
+
},
|
|
52
74
|
};
|
|
53
75
|
}
|
|
54
76
|
export function createEvalCheckContext(input) {
|
package/esm/src/eval/index.d.ts
CHANGED
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
* ]),
|
|
15
15
|
* metrics: [
|
|
16
16
|
* metrics.answer.contains({ text: "Paris" }).gate(),
|
|
17
|
+
* metrics.agent.calledTool("search_docs").gate(),
|
|
17
18
|
* metrics.agent.noFailedTools().gate(),
|
|
18
19
|
* ],
|
|
19
20
|
* });
|
|
@@ -51,6 +52,6 @@ export { runEval } from "./runner.js";
|
|
|
51
52
|
export { deriveEvalId, discoverEvals, findEvalById } from "./discovery.js";
|
|
52
53
|
export { createEvalSourceDocument, getEvalEditableFieldSchema, getEvalRunSchema, getEvalSourceDocumentSchema, getEvalSourcePatchSchema, getEvalSourceReferenceSchema, getEvalStudioCapabilitySchema, } from "./studio.js";
|
|
53
54
|
export type { DiscoveredEval, EvalDiscoveryOptions, EvalDiscoveryResult } from "./discovery.js";
|
|
54
|
-
export type { EvalAgentAdapter, EvalAgentAdapterContext, EvalAgentAdapterResult, EvalAgentInput, EvalCheckContext, EvalDataset, EvalDatasetLoadContext, EvalDefinition, EvalDurationSummary, EvalExample, EvalExampleInput, EvalExpect, EvalExpectation, EvalFailedExampleSummary, EvalFlakeSummary, EvalGateFailureSummary, EvalMetric, EvalMetricContext, EvalMetricDeltaSummary, EvalMetricFamily, EvalMetricResult, EvalMetricSummary, EvalMetricThreshold, EvalRecord, EvalReport, EvalReportComparison, EvalReportExportConfig, EvalReportSummary, EvalSeverity, EvalSource, EvalTargetKind, EvalToolCall, EvalTrace, EvalUsage, EvalUsageSummary, RunEvalOptions, } from "./types.js";
|
|
55
|
+
export type { EvalAgentAdapter, EvalAgentAdapterContext, EvalAgentAdapterResult, EvalAgentInput, EvalCheckContext, EvalDataset, EvalDatasetLoadContext, EvalDefinition, EvalDurationSummary, EvalExample, EvalExampleInput, EvalExpect, EvalExpectation, EvalFailedExampleSummary, EvalFlakeSummary, EvalGateFailureSummary, EvalMetric, EvalMetricContext, EvalMetricDeltaSummary, EvalMetricFamily, EvalMetricResult, EvalMetricSummary, EvalMetricThreshold, EvalRecord, EvalReport, EvalReportComparison, EvalReportExportConfig, EvalReportSummary, EvalSeverity, EvalSource, EvalTargetKind, EvalToolCall, EvalToolCallCountOptions, EvalToolCallMatchOptions, EvalToolCallStatus, EvalToolInputMatchMode, EvalTrace, EvalUsage, EvalUsageSummary, RunEvalOptions, } from "./types.js";
|
|
55
56
|
export type { CreateEvalSourceDocumentOptions, EvalEditableField, EvalRun, EvalSourceDocument, EvalSourcePatch, EvalSourceReference, EvalStudioCapability, } from "./studio.js";
|
|
56
57
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/eval/index.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AACH,OAAO,yBAAyB,CAAC;AAGjC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC3E,OAAO,EACL,wBAAwB,EACxB,0BAA0B,EAC1B,gBAAgB,EAChB,2BAA2B,EAC3B,wBAAwB,EACxB,4BAA4B,EAC5B,6BAA6B,GAC9B,MAAM,aAAa,CAAC;AAErB,YAAY,EAAE,cAAc,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAChG,YAAY,EACV,gBAAgB,EAChB,uBAAuB,EACvB,sBAAsB,EACtB,cAAc,EACd,gBAAgB,EAChB,WAAW,EACX,sBAAsB,EACtB,cAAc,EACd,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,UAAU,EACV,eAAe,EACf,wBAAwB,EACxB,gBAAgB,EAChB,sBAAsB,EACtB,UAAU,EACV,iBAAiB,EACjB,sBAAsB,EACtB,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,EACnB,UAAU,EACV,UAAU,EACV,oBAAoB,EACpB,sBAAsB,EACtB,iBAAiB,EACjB,YAAY,EACZ,UAAU,EACV,cAAc,EACd,YAAY,EACZ,wBAAwB,EACxB,wBAAwB,EACxB,kBAAkB,EAClB,sBAAsB,EACtB,SAAS,EACT,SAAS,EACT,gBAAgB,EAChB,cAAc,GACf,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,+BAA+B,EAC/B,iBAAiB,EACjB,OAAO,EACP,kBAAkB,EAClB,eAAe,EACf,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,aAAa,CAAC"}
|
package/esm/src/eval/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { EvalMetric } from "./types.js";
|
|
1
|
+
import type { EvalMetric, EvalToolCallCountOptions, EvalToolCallMatchOptions } from "./types.js";
|
|
2
2
|
type JudgeRubricInput = {
|
|
3
3
|
rubric: string;
|
|
4
4
|
judge?: (input: {
|
|
@@ -33,6 +33,9 @@ export declare const metrics: {
|
|
|
33
33
|
};
|
|
34
34
|
readonly agent: {
|
|
35
35
|
readonly noFailedTools: () => EvalMetric;
|
|
36
|
+
readonly calledTool: (name: string, options?: EvalToolCallMatchOptions) => EvalMetric;
|
|
37
|
+
readonly notCalledTool: (name: string) => EvalMetric;
|
|
38
|
+
readonly toolCallCount: (name: string, options: EvalToolCallCountOptions) => EvalMetric;
|
|
36
39
|
};
|
|
37
40
|
readonly ops: {
|
|
38
41
|
readonly latency: (options: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../../src/src/eval/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,UAAU,
|
|
1
|
+
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../../src/src/eval/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,UAAU,EAMV,wBAAwB,EACxB,wBAAwB,EACzB,MAAM,YAAY,CAAC;AAUpB,KAAK,gBAAgB,GAAG;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE;QACd,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,OAAO,CAAC;QACf,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAChC,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACnC,KAAK,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACxE,CAAC;AAEF,iBAAS,aAAa,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,CAQ9C;AASD,iBAAS,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAE/C;AAuFD,0FAA0F;AAC1F,eAAO,MAAM,OAAO;;mCAEF,UAAU;qCAON;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,aAAa,CAAC,EAAE,OAAO,CAAA;SAAE,KAAG,UAAU;kCAWzD;YAAE,OAAO,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAA;SAAE,KAAG,UAAU;sCAY5C;YAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;SAAE,KAAG,UAAU;;;sCAWrC,UAAU;oCAgBV,MAAM,YAAY,wBAAwB,KAAG,UAAU;uCAcpD,MAAM,KAAG,UAAU;uCAcnB,MAAM,WAAW,wBAAwB,KAAG,UAAU;;;oCAgBzD;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,KAAG,UAAU;mCAc/B;YAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;YAAC,SAAS,CAAC,EAAE,MAAM,CAAA;SAAE,KAAG,UAAU;iCAoB3E;YAAE,MAAM,EAAE,MAAM,CAAA;SAAE,KAAG,UAAU;;;mCAiB7B,gBAAgB,KAAG,UAAU;;CAmCvC,CAAC;AAEX,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,CAAC"}
|
package/esm/src/eval/metrics.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { evaluateCalledTool, evaluateNotCalledTool, evaluateToolCallCount, isEvalToolFailed, } from "./tool-behavior.js";
|
|
1
2
|
function getOutputText(output) {
|
|
2
3
|
if (typeof output === "string")
|
|
3
4
|
return output;
|
|
@@ -85,9 +86,6 @@ function createMetric(name, family, evaluator, config) {
|
|
|
85
86
|
function scoreResult(name, family, severity, pass, score = pass ? 1 : 0) {
|
|
86
87
|
return { name, family, severity, score, pass };
|
|
87
88
|
}
|
|
88
|
-
function isToolFailed(toolCall) {
|
|
89
|
-
return toolCall.status === "error" || typeof toolCall.error === "string";
|
|
90
|
-
}
|
|
91
89
|
/** Metric factories for deterministic answers, agent behavior, operations, and judges. */
|
|
92
90
|
export const metrics = {
|
|
93
91
|
answer: {
|
|
@@ -125,7 +123,7 @@ export const metrics = {
|
|
|
125
123
|
agent: {
|
|
126
124
|
noFailedTools() {
|
|
127
125
|
return createMetric("agent.noFailedTools", "agent", (record) => {
|
|
128
|
-
const failedTools = record.trace.toolCalls.filter(
|
|
126
|
+
const failedTools = record.trace.toolCalls.filter(isEvalToolFailed).map((tool) => tool.name);
|
|
129
127
|
return {
|
|
130
128
|
name: "agent.noFailedTools",
|
|
131
129
|
family: "agent",
|
|
@@ -136,6 +134,30 @@ export const metrics = {
|
|
|
136
134
|
};
|
|
137
135
|
});
|
|
138
136
|
},
|
|
137
|
+
calledTool(name, options) {
|
|
138
|
+
return createMetric("agent.calledTool", "agent", (record) => ({
|
|
139
|
+
name: "agent.calledTool",
|
|
140
|
+
family: "agent",
|
|
141
|
+
severity: "gate",
|
|
142
|
+
...evaluateCalledTool(record, name, options),
|
|
143
|
+
}), { tool: name, ...(options ?? {}) });
|
|
144
|
+
},
|
|
145
|
+
notCalledTool(name) {
|
|
146
|
+
return createMetric("agent.notCalledTool", "agent", (record) => ({
|
|
147
|
+
name: "agent.notCalledTool",
|
|
148
|
+
family: "agent",
|
|
149
|
+
severity: "gate",
|
|
150
|
+
...evaluateNotCalledTool(record, name),
|
|
151
|
+
}), { tool: name });
|
|
152
|
+
},
|
|
153
|
+
toolCallCount(name, options) {
|
|
154
|
+
return createMetric("agent.toolCallCount", "agent", (record) => ({
|
|
155
|
+
name: "agent.toolCallCount",
|
|
156
|
+
family: "agent",
|
|
157
|
+
severity: "gate",
|
|
158
|
+
...evaluateToolCallCount(record, name, options),
|
|
159
|
+
}), { tool: name, ...options });
|
|
160
|
+
},
|
|
139
161
|
},
|
|
140
162
|
ops: {
|
|
141
163
|
latency(options) {
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { EvalMetricResult, EvalRecord, EvalToolCall, EvalToolCallCountOptions, EvalToolCallMatchOptions } from "./types.js";
|
|
2
|
+
type ToolBehaviorResult = Omit<EvalMetricResult, "name" | "family" | "severity">;
|
|
3
|
+
export declare function findEvalToolCalls(record: EvalRecord, name: string): EvalToolCall[];
|
|
4
|
+
export declare function isEvalToolFailed(toolCall: {
|
|
5
|
+
status?: string;
|
|
6
|
+
error?: string;
|
|
7
|
+
}): boolean;
|
|
8
|
+
export declare function evaluateCalledTool(record: EvalRecord, name: string, options?: EvalToolCallMatchOptions): ToolBehaviorResult;
|
|
9
|
+
export declare function evaluateNotCalledTool(record: EvalRecord, name: string): ToolBehaviorResult;
|
|
10
|
+
export declare function evaluateToolCallCount(record: EvalRecord, name: string, options: EvalToolCallCountOptions): ToolBehaviorResult;
|
|
11
|
+
export {};
|
|
12
|
+
//# sourceMappingURL=tool-behavior.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tool-behavior.d.ts","sourceRoot":"","sources":["../../../src/src/eval/tool-behavior.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,UAAU,EACV,YAAY,EACZ,wBAAwB,EACxB,wBAAwB,EAEzB,MAAM,YAAY,CAAC;AAEpB,KAAK,kBAAkB,GAAG,IAAI,CAAC,gBAAgB,EAAE,MAAM,GAAG,QAAQ,GAAG,UAAU,CAAC,CAAC;AAkEjF,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,YAAY,EAAE,CAElF;AAED,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAGvF;AAED,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,UAAU,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,wBAA6B,GACrC,kBAAkB,CAuBpB;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,kBAAkB,CAQ1F;AAED,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,UAAU,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,wBAAwB,GAChC,kBAAkB,CAYpB"}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
function sortJsonValue(value) {
|
|
2
|
+
if (Array.isArray(value))
|
|
3
|
+
return value.map(sortJsonValue);
|
|
4
|
+
if (!value || typeof value !== "object")
|
|
5
|
+
return value;
|
|
6
|
+
const sorted = {};
|
|
7
|
+
for (const key of Object.keys(value).sort()) {
|
|
8
|
+
sorted[key] = sortJsonValue(value[key]);
|
|
9
|
+
}
|
|
10
|
+
return sorted;
|
|
11
|
+
}
|
|
12
|
+
function stableStringify(value) {
|
|
13
|
+
return JSON.stringify(sortJsonValue(value));
|
|
14
|
+
}
|
|
15
|
+
function valuesEqual(actual, expected) {
|
|
16
|
+
return stableStringify(actual) === stableStringify(expected);
|
|
17
|
+
}
|
|
18
|
+
function isRecord(value) {
|
|
19
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
20
|
+
}
|
|
21
|
+
function partialMatch(actual, expected) {
|
|
22
|
+
if (Array.isArray(expected)) {
|
|
23
|
+
return Array.isArray(actual) &&
|
|
24
|
+
expected.every((entry, index) => partialMatch(actual[index], entry));
|
|
25
|
+
}
|
|
26
|
+
if (isRecord(expected)) {
|
|
27
|
+
if (!isRecord(actual))
|
|
28
|
+
return false;
|
|
29
|
+
return Object.entries(expected).every(([key, value]) => partialMatch(actual[key], value));
|
|
30
|
+
}
|
|
31
|
+
return valuesEqual(actual, expected);
|
|
32
|
+
}
|
|
33
|
+
function matchesInput(actual, expected, mode) {
|
|
34
|
+
return mode === "exact" ? valuesEqual(actual, expected) : partialMatch(actual, expected);
|
|
35
|
+
}
|
|
36
|
+
function hasExpectedInput(options) {
|
|
37
|
+
return Object.hasOwn(options, "input");
|
|
38
|
+
}
|
|
39
|
+
function expectedCount(options) {
|
|
40
|
+
return {
|
|
41
|
+
...(options.exact !== undefined ? { exact: options.exact } : {}),
|
|
42
|
+
...(options.min !== undefined ? { min: options.min } : {}),
|
|
43
|
+
...(options.max !== undefined ? { max: options.max } : {}),
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
function countPasses(count, options) {
|
|
47
|
+
if (options.exact !== undefined)
|
|
48
|
+
return count === options.exact;
|
|
49
|
+
const minPass = options.min === undefined || count >= options.min;
|
|
50
|
+
const maxPass = options.max === undefined || count <= options.max;
|
|
51
|
+
return minPass && maxPass;
|
|
52
|
+
}
|
|
53
|
+
export function findEvalToolCalls(record, name) {
|
|
54
|
+
return record.trace.toolCalls.filter((tool) => tool.name === name);
|
|
55
|
+
}
|
|
56
|
+
export function isEvalToolFailed(toolCall) {
|
|
57
|
+
return toolCall.status === "error" || toolCall.status === "denied" ||
|
|
58
|
+
typeof toolCall.error === "string";
|
|
59
|
+
}
|
|
60
|
+
export function evaluateCalledTool(record, name, options = {}) {
|
|
61
|
+
const calls = findEvalToolCalls(record, name);
|
|
62
|
+
const expectedInputProvided = hasExpectedInput(options);
|
|
63
|
+
const match = options.match ?? "partial";
|
|
64
|
+
const inputMatched = !expectedInputProvided ||
|
|
65
|
+
calls.some((call) => matchesInput(call.input, options.input, match));
|
|
66
|
+
const pass = calls.length > 0 && inputMatched;
|
|
67
|
+
const evidence = {
|
|
68
|
+
tool: name,
|
|
69
|
+
calls: calls.length,
|
|
70
|
+
};
|
|
71
|
+
if (expectedInputProvided) {
|
|
72
|
+
evidence.expectedInput = options.input;
|
|
73
|
+
evidence.match = match;
|
|
74
|
+
if (!inputMatched)
|
|
75
|
+
evidence.actualInputs = calls.map((call) => call.input);
|
|
76
|
+
}
|
|
77
|
+
return {
|
|
78
|
+
score: pass ? 1 : 0,
|
|
79
|
+
pass,
|
|
80
|
+
evidence,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
export function evaluateNotCalledTool(record, name) {
|
|
84
|
+
const calls = findEvalToolCalls(record, name);
|
|
85
|
+
const pass = calls.length === 0;
|
|
86
|
+
return {
|
|
87
|
+
score: pass ? 1 : 0,
|
|
88
|
+
pass,
|
|
89
|
+
evidence: { tool: name, calls: calls.length },
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
export function evaluateToolCallCount(record, name, options) {
|
|
93
|
+
const calls = findEvalToolCalls(record, name);
|
|
94
|
+
const pass = countPasses(calls.length, options);
|
|
95
|
+
return {
|
|
96
|
+
score: pass ? 1 : 0,
|
|
97
|
+
pass,
|
|
98
|
+
evidence: {
|
|
99
|
+
tool: name,
|
|
100
|
+
calls: calls.length,
|
|
101
|
+
expected: expectedCount(options),
|
|
102
|
+
},
|
|
103
|
+
};
|
|
104
|
+
}
|
package/esm/src/eval/types.d.ts
CHANGED
|
@@ -49,10 +49,28 @@ export interface EvalUsage {
|
|
|
49
49
|
totalTokens?: number;
|
|
50
50
|
costUsd?: number;
|
|
51
51
|
}
|
|
52
|
+
/** Normalized status for a tool call captured during an eval record. */
|
|
53
|
+
export type EvalToolCallStatus = "ok" | "error" | "skipped" | "denied";
|
|
54
|
+
/** How expected tool input is compared to the captured tool input. */
|
|
55
|
+
export type EvalToolInputMatchMode = "exact" | "partial";
|
|
56
|
+
/** Options for matching a required tool call. */
|
|
57
|
+
export interface EvalToolCallMatchOptions {
|
|
58
|
+
input?: unknown;
|
|
59
|
+
match?: EvalToolInputMatchMode;
|
|
60
|
+
}
|
|
61
|
+
/** Options for checking how often a tool was called. */
|
|
62
|
+
export interface EvalToolCallCountOptions {
|
|
63
|
+
exact?: number;
|
|
64
|
+
min?: number;
|
|
65
|
+
max?: number;
|
|
66
|
+
}
|
|
52
67
|
/** Tool call metadata captured during one eval record. */
|
|
53
68
|
export interface EvalToolCall {
|
|
69
|
+
id?: string;
|
|
54
70
|
name: string;
|
|
55
|
-
status?:
|
|
71
|
+
status?: EvalToolCallStatus;
|
|
72
|
+
input?: unknown;
|
|
73
|
+
output?: unknown;
|
|
56
74
|
error?: string;
|
|
57
75
|
metadata?: Record<string, unknown>;
|
|
58
76
|
}
|
|
@@ -118,6 +136,9 @@ export interface EvalExpect {
|
|
|
118
136
|
completed(): EvalExpectation;
|
|
119
137
|
outputContains(text: string): EvalExpectation;
|
|
120
138
|
noFailedTools(): EvalExpectation;
|
|
139
|
+
calledTool(name: string, options?: EvalToolCallMatchOptions): EvalExpectation;
|
|
140
|
+
notCalledTool(name: string): EvalExpectation;
|
|
141
|
+
toolCallCount(name: string, options: EvalToolCallCountOptions): EvalExpectation;
|
|
121
142
|
}
|
|
122
143
|
/** Context passed to an eval definition's `check` callback. */
|
|
123
144
|
export interface EvalCheckContext {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/src/eval/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EACV,uBAAuB,EACvB,0BAA0B,EAC1B,sBAAsB,EACvB,MAAM,6BAA6B,CAAC;AAErC,qEAAqE;AACrE,MAAM,MAAM,cAAc,GAAG,OAAO,CAAC;AAErC,yDAAyD;AACzD,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAC;AAEtD,wDAAwD;AACxD,MAAM,MAAM,gBAAgB,GAAG,QAAQ,GAAG,OAAO,GAAG,KAAK,GAAG,OAAO,GAAG,OAAO,CAAC;AAE9E,yDAAyD;AACzD,MAAM,MAAM,mBAAmB,GAAG;IAChC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,gEAAgE;AAChE,MAAM,MAAM,gBAAgB,CAAC,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;AAEjD,mEAAmE;AACnE,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,OAAO,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,0DAA0D;AAC1D,MAAM,MAAM,gBAAgB,GAAG;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,OAAO,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,CAAC;AAEF,yCAAyC;AACzC,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,iDAAiD;AACjD,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,OAAO,CAAC;IAClC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,WAAW,EAAE,CAAC;IACzB,IAAI,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;CAC/D;AAED,yDAAyD;AACzD,MAAM,WAAW,SAAS;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,0DAA0D;AAC1D,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/src/eval/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EACV,uBAAuB,EACvB,0BAA0B,EAC1B,sBAAsB,EACvB,MAAM,6BAA6B,CAAC;AAErC,qEAAqE;AACrE,MAAM,MAAM,cAAc,GAAG,OAAO,CAAC;AAErC,yDAAyD;AACzD,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAC;AAEtD,wDAAwD;AACxD,MAAM,MAAM,gBAAgB,GAAG,QAAQ,GAAG,OAAO,GAAG,KAAK,GAAG,OAAO,GAAG,OAAO,CAAC;AAE9E,yDAAyD;AACzD,MAAM,MAAM,mBAAmB,GAAG;IAChC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,gEAAgE;AAChE,MAAM,MAAM,gBAAgB,CAAC,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;AAEjD,mEAAmE;AACnE,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,OAAO,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,0DAA0D;AAC1D,MAAM,MAAM,gBAAgB,GAAG;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,OAAO,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,CAAC;AAEF,yCAAyC;AACzC,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,iDAAiD;AACjD,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,OAAO,CAAC;IAClC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,WAAW,EAAE,CAAC;IACzB,IAAI,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;CAC/D;AAED,yDAAyD;AACzD,MAAM,WAAW,SAAS;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,wEAAwE;AACxE,MAAM,MAAM,kBAAkB,GAAG,IAAI,GAAG,OAAO,GAAG,SAAS,GAAG,QAAQ,CAAC;AAEvE,sEAAsE;AACtE,MAAM,MAAM,sBAAsB,GAAG,OAAO,GAAG,SAAS,CAAC;AAEzD,iDAAiD;AACjD,MAAM,WAAW,wBAAwB;IACvC,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,sBAAsB,CAAC;CAChC;AAED,wDAAwD;AACxD,MAAM,WAAW,wBAAwB;IACvC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,0DAA0D;AAC1D,MAAM,WAAW,YAAY;IAC3B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,mDAAmD;AACnD,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,OAAO,EAAE,CAAC;IAClB,SAAS,EAAE,YAAY,EAAE,CAAC;CAC3B;AAED,iEAAiE;AACjE,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC7B,MAAM,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC7B;AAED,qDAAqD;AACrD,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,4DAA4D;AAC5D,MAAM,WAAW,iBAAiB;IAChC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAClB;AAED,gDAAgD;AAChD,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,SAAS,CAAC,EAAE,mBAAmB,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,gBAAgB,CAAC,gBAAgB,CAAC,CAAC;IAC9F,IAAI,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,UAAU,CAAC;IAClD,IAAI,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,UAAU,CAAC;IAClD,MAAM,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,UAAU,CAAC;CACrD;AAED,wDAAwD;AACxD,MAAM,WAAW,eAAe;IAC9B,IAAI,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,gBAAgB,CAAC;IACxD,IAAI,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,gBAAgB,CAAC;IACxD,MAAM,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,gBAAgB,CAAC;CAC3D;AAED,6DAA6D;AAC7D,MAAM,WAAW,UAAU;IACzB,SAAS,IAAI,eAAe,CAAC;IAC7B,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,CAAC;IAC9C,aAAa,IAAI,eAAe,CAAC;IACjC,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,wBAAwB,GAAG,eAAe,CAAC;IAC9E,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,CAAC;IAC7C,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,wBAAwB,GAAG,eAAe,CAAC;CACjF;AAED,+DAA+D;AAC/D,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;CACpB;AAED,wDAAwD;AACxD,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,kEAAkE;AAClE,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,WAAW,CAAC;IACrB,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,gBAAgB,KAAK,gBAAgB,CAAC,IAAI,CAAC,CAAC;CAC/D;AAED,qCAAqC;AACrC,MAAM,WAAW,cAAc;IAC7B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,WAAW,GAAG,gBAAgB,EAAE,CAAC;IAC1C,OAAO,CAAC,EAAE,UAAU,EAAE,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,gBAAgB,KAAK,gBAAgB,CAAC,IAAI,CAAC,CAAC;CAC/D;AAED,6EAA6E;AAC7E,MAAM,WAAW,uBAAuB;IACtC,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,2DAA2D;AAC3D,MAAM,WAAW,sBAAsB;IACrC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAC3B,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,6DAA6D;AAC7D,MAAM,MAAM,gBAAgB,GAAG,CAC7B,OAAO,EAAE,uBAAuB,KAC7B,gBAAgB,CAAC,MAAM,GAAG,sBAAsB,CAAC,CAAC;AAEvD,2CAA2C;AAC3C,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE;QACR,KAAK,EAAE,gBAAgB,CAAC;KACzB,CAAC;IACF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IACjB,MAAM,CAAC,EAAE,sBAAsB,CAAC;CACjC;AAED,wDAAwD;AACxD,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,EAAE,0BAA0B,CAAC;IACtC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,OAAO,CAAC,EAAE,uBAAuB,CAAC;CACnC;AAED,kDAAkD;AAClD,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,6CAA6C;AAC7C,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED,uCAAuC;AACvC,MAAM,WAAW,gBAAgB;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qDAAqD;AACrD,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,kEAAkE;AAClE,MAAM,WAAW,wBAAwB;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,OAAO,CAAC;CAChB;AAED,uDAAuD;AACvD,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,4EAA4E;AAC5E,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,qDAAqD;AACrD,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,wBAAwB,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,sBAAsB,EAAE,CAAC;IACvC,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,uDAAuD;AACvD,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,iDAAiD;IACjD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,yCAAyC;IACzC,QAAQ,CAAC,EAAE,mBAAmB,CAAC;IAC/B,mCAAmC;IACnC,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,uEAAuE;IACvE,YAAY,CAAC,EAAE,sBAAsB,EAAE,CAAC;IACxC,oEAAoE;IACpE,cAAc,CAAC,EAAE,wBAAwB,EAAE,CAAC;IAC5C,6CAA6C;IAC7C,MAAM,CAAC,EAAE,gBAAgB,CAAC;CAC3B;AAED,sDAAsD;AACtD,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,cAAc,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,OAAO,CAAC,EAAE,sBAAsB,EAAE,CAAC;CACpC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agent-stream.handler.d.ts","sourceRoot":"","sources":["../../../../../src/src/server/handlers/request/agent-stream.handler.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,KAAK,yBAAyB,EAAE,MAAM,oCAAoC,CAAC;AACpF,OAAO,EAEL,KAAK,+BAA+B,EACrC,MAAM,wCAAwC,CAAC;AAOhD,OAAO,EACL,4BAA4B,EAE7B,MAAM,2CAA2C,CAAC;
|
|
1
|
+
{"version":3,"file":"agent-stream.handler.d.ts","sourceRoot":"","sources":["../../../../../src/src/server/handlers/request/agent-stream.handler.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,KAAK,yBAAyB,EAAE,MAAM,oCAAoC,CAAC;AACpF,OAAO,EAEL,KAAK,+BAA+B,EACrC,MAAM,wCAAwC,CAAC;AAOhD,OAAO,EACL,4BAA4B,EAE7B,MAAM,2CAA2C,CAAC;AAyBnD,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAmB,aAAa,EAAE,MAAM,aAAa,CAAC;AAanG,MAAM,WAAW,sBACf,SAAQ,yBAAyB,EAAE,+BAA+B;IAClE,4BAA4B,CAAC,EAAE,OAAO,4BAA4B,CAAC;CACpE;AAqdD,qBAAa,kBAAmB,SAAQ,WAAW;IASrC,OAAO,CAAC,QAAQ,CAAC,IAAI;IARjC,QAAQ,EAAE,eAAe,CAMvB;gBAE2B,IAAI,GAAE,sBAAoC;IAIvE,OAAO,CAAC,sBAAsB;IAwBxB,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;CAoLxE"}
|
|
@@ -108,6 +108,58 @@ function getRequestedStudioToolNames(input) {
|
|
|
108
108
|
.filter((toolName) => STUDIO_RUNTIME_REMOTE_TOOL_NAMES.has(toolName))
|
|
109
109
|
.sort();
|
|
110
110
|
}
|
|
111
|
+
function sanitizeForwardedRuntimeAllowedTools(input) {
|
|
112
|
+
const forwardedProps = input.forwardedProps;
|
|
113
|
+
if (!isRecord(forwardedProps)) {
|
|
114
|
+
return forwardedProps;
|
|
115
|
+
}
|
|
116
|
+
const runtimeOverrides = isRecord(forwardedProps.runtimeOverrides)
|
|
117
|
+
? forwardedProps.runtimeOverrides
|
|
118
|
+
: null;
|
|
119
|
+
if (!runtimeOverrides || !Object.hasOwn(runtimeOverrides, "allowedTools")) {
|
|
120
|
+
return forwardedProps;
|
|
121
|
+
}
|
|
122
|
+
const allowedTools = runtimeOverrides.allowedTools;
|
|
123
|
+
if (!Array.isArray(allowedTools) || !allowedTools.every((toolName) => typeof toolName === "string")) {
|
|
124
|
+
return forwardedProps;
|
|
125
|
+
}
|
|
126
|
+
const availableToolNames = new Set(input.availableToolNames);
|
|
127
|
+
// Platform remote tools are gated separately by the child agent config in
|
|
128
|
+
// withVeryfrontPlatformRemoteTools. The Studio path is the one that consumes
|
|
129
|
+
// forwarded allowedTools, and Studio-only runtime tools are preserved only
|
|
130
|
+
// for trusted Studio clients that can already attach the Studio MCP surface.
|
|
131
|
+
const sanitizedAllowedTools = allowedTools.filter((toolName) => availableToolNames.has(toolName) ||
|
|
132
|
+
(input.allowStudioRuntimeTools && STUDIO_RUNTIME_REMOTE_TOOL_NAMES.has(toolName)));
|
|
133
|
+
if (sanitizedAllowedTools.length === allowedTools.length) {
|
|
134
|
+
return forwardedProps;
|
|
135
|
+
}
|
|
136
|
+
const nextRuntimeOverrides = {
|
|
137
|
+
...runtimeOverrides,
|
|
138
|
+
allowedTools: sanitizedAllowedTools,
|
|
139
|
+
};
|
|
140
|
+
if (sanitizedAllowedTools.length === 0) {
|
|
141
|
+
delete nextRuntimeOverrides.allowedTools;
|
|
142
|
+
}
|
|
143
|
+
const nextForwardedProps = {
|
|
144
|
+
...forwardedProps,
|
|
145
|
+
runtimeOverrides: nextRuntimeOverrides,
|
|
146
|
+
};
|
|
147
|
+
if (Object.keys(nextRuntimeOverrides).length === 0) {
|
|
148
|
+
delete nextForwardedProps.runtimeOverrides;
|
|
149
|
+
}
|
|
150
|
+
return Object.keys(nextForwardedProps).length > 0 ? nextForwardedProps : undefined;
|
|
151
|
+
}
|
|
152
|
+
function sanitizeRuntimeRunAgentInput(input) {
|
|
153
|
+
const clientProfile = resolveRuntimeClientProfile(input.forwardedProps);
|
|
154
|
+
return {
|
|
155
|
+
...input,
|
|
156
|
+
forwardedProps: sanitizeForwardedRuntimeAllowedTools({
|
|
157
|
+
forwardedProps: input.forwardedProps,
|
|
158
|
+
availableToolNames: input.tools.map((tool) => tool.name),
|
|
159
|
+
allowStudioRuntimeTools: clientAllowsStudioMcp(clientProfile),
|
|
160
|
+
}),
|
|
161
|
+
};
|
|
162
|
+
}
|
|
111
163
|
function getVeryfrontApiMcpPolicy(agent) {
|
|
112
164
|
const requestedToolNames = new Set();
|
|
113
165
|
const deniedToolNames = new Set();
|
|
@@ -368,7 +420,7 @@ export class AgentStreamHandler extends BaseHandler {
|
|
|
368
420
|
});
|
|
369
421
|
return this.respond(builder.json({ error: "Agent not found" }, 404));
|
|
370
422
|
}
|
|
371
|
-
const runtimeInput = toRuntimeRunAgentInput(payload);
|
|
423
|
+
const runtimeInput = sanitizeRuntimeRunAgentInput(toRuntimeRunAgentInput(payload));
|
|
372
424
|
const apiAuthToken = payload.credentials?.authToken || ctx.proxyToken ||
|
|
373
425
|
getHostEnv("VERYFRONT_API_TOKEN") || "";
|
|
374
426
|
const platformRuntimeAgent = await withVeryfrontPlatformRemoteTools({
|