veryfront 0.1.877 → 0.1.878
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/deno.js +1 -1
- package/esm/src/eval/agent-service/durable-run-canaries/cli-runner.d.ts +21 -0
- package/esm/src/eval/agent-service/durable-run-canaries/cli-runner.d.ts.map +1 -0
- package/esm/src/eval/agent-service/durable-run-canaries/cli-runner.js +66 -0
- package/esm/src/eval/agent-service/durable-run-canaries/environment.d.ts +14 -0
- package/esm/src/eval/agent-service/durable-run-canaries/environment.d.ts.map +1 -0
- package/esm/src/eval/agent-service/durable-run-canaries/environment.js +15 -0
- package/esm/src/eval/agent-service/durable-run-canaries/index.d.ts +5 -0
- package/esm/src/eval/agent-service/durable-run-canaries/index.d.ts.map +1 -0
- package/esm/src/eval/agent-service/durable-run-canaries/index.js +4 -0
- package/esm/src/eval/agent-service/durable-run-canaries/runner.d.ts +118 -0
- package/esm/src/eval/agent-service/durable-run-canaries/runner.d.ts.map +1 -0
- package/esm/src/eval/agent-service/durable-run-canaries/runner.js +391 -0
- package/esm/src/eval/agent-service/durable-run-canaries/validation.d.ts +12 -0
- package/esm/src/eval/agent-service/durable-run-canaries/validation.d.ts.map +1 -0
- package/esm/src/eval/agent-service/durable-run-canaries/validation.js +71 -0
- package/esm/src/eval/agent-service/live-evals/api-client.d.ts +102 -0
- package/esm/src/eval/agent-service/live-evals/api-client.d.ts.map +1 -0
- package/esm/src/eval/agent-service/live-evals/api-client.js +313 -0
- package/esm/src/eval/agent-service/live-evals/cli-runner.d.ts +40 -0
- package/esm/src/eval/agent-service/live-evals/cli-runner.d.ts.map +1 -0
- package/esm/src/eval/agent-service/live-evals/cli-runner.js +144 -0
- package/esm/src/eval/agent-service/live-evals/environment.d.ts +15 -0
- package/esm/src/eval/agent-service/live-evals/environment.d.ts.map +1 -0
- package/esm/src/eval/agent-service/live-evals/environment.js +20 -0
- package/esm/src/eval/agent-service/live-evals/formatting.d.ts +15 -0
- package/esm/src/eval/agent-service/live-evals/formatting.d.ts.map +1 -0
- package/esm/src/eval/agent-service/live-evals/formatting.js +75 -0
- package/esm/src/eval/agent-service/live-evals/index.d.ts +12 -0
- package/esm/src/eval/agent-service/live-evals/index.d.ts.map +1 -0
- package/esm/src/eval/agent-service/live-evals/index.js +11 -0
- package/esm/src/eval/agent-service/live-evals/metadata.d.ts +34 -0
- package/esm/src/eval/agent-service/live-evals/metadata.d.ts.map +1 -0
- package/esm/src/eval/agent-service/live-evals/metadata.js +116 -0
- package/esm/src/eval/agent-service/live-evals/performance.d.ts +19 -0
- package/esm/src/eval/agent-service/live-evals/performance.d.ts.map +1 -0
- package/esm/src/eval/agent-service/live-evals/performance.js +22 -0
- package/esm/src/eval/agent-service/live-evals/preflight.d.ts +12 -0
- package/esm/src/eval/agent-service/live-evals/preflight.d.ts.map +1 -0
- package/esm/src/eval/agent-service/live-evals/preflight.js +27 -0
- package/esm/src/eval/agent-service/live-evals/report.d.ts +55 -0
- package/esm/src/eval/agent-service/live-evals/report.d.ts.map +1 -0
- package/esm/src/eval/agent-service/live-evals/report.js +79 -0
- package/esm/src/eval/agent-service/live-evals/request.d.ts +31 -0
- package/esm/src/eval/agent-service/live-evals/request.d.ts.map +1 -0
- package/esm/src/eval/agent-service/live-evals/request.js +48 -0
- package/esm/src/eval/agent-service/live-evals/result.d.ts +56 -0
- package/esm/src/eval/agent-service/live-evals/result.d.ts.map +1 -0
- package/esm/src/eval/agent-service/live-evals/result.js +36 -0
- package/esm/src/eval/agent-service/live-evals/runner.d.ts +135 -0
- package/esm/src/eval/agent-service/live-evals/runner.d.ts.map +1 -0
- package/esm/src/eval/agent-service/live-evals/runner.js +396 -0
- package/esm/src/eval/agent-service.d.ts +2 -0
- package/esm/src/eval/agent-service.d.ts.map +1 -1
- package/esm/src/eval/agent-service.js +2 -0
- package/esm/src/utils/version-constant.d.ts +1 -1
- package/esm/src/utils/version-constant.js +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/** Check whether every live eval tag is present. */
|
|
2
|
+
export function hasEveryLiveEvalTag(tags, requestedTags) {
|
|
3
|
+
for (const requestedTag of requestedTags) {
|
|
4
|
+
if (!tags.includes(requestedTag)) {
|
|
5
|
+
return false;
|
|
6
|
+
}
|
|
7
|
+
}
|
|
8
|
+
return true;
|
|
9
|
+
}
|
|
10
|
+
/** Builds live eval case tag summary. */
|
|
11
|
+
export function buildLiveEvalCaseTagSummary(cases) {
|
|
12
|
+
const counts = new Map();
|
|
13
|
+
for (const testCase of cases) {
|
|
14
|
+
const caseTags = testCase.metadata?.tags ?? [];
|
|
15
|
+
for (const tag of caseTags) {
|
|
16
|
+
counts.set(tag, (counts.get(tag) ?? 0) + 1);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
return Object.fromEntries([...counts.entries()].sort(([left], [right]) => left.localeCompare(right)));
|
|
20
|
+
}
|
|
21
|
+
/** Select live eval cases helper. */
|
|
22
|
+
export function selectLiveEvalCases(input) {
|
|
23
|
+
const configuredCases = input.runWriteEvals
|
|
24
|
+
? [
|
|
25
|
+
...input.readOnlyCases,
|
|
26
|
+
...input.writeCases,
|
|
27
|
+
...(input.runExperimentalWriteEvals ? input.experimentalWriteCases : []),
|
|
28
|
+
]
|
|
29
|
+
: input.readOnlyCases;
|
|
30
|
+
const selectedCases = input.requestedCaseIds.size > 0
|
|
31
|
+
? input.allCases.filter((testCase) => input.requestedCaseIds.has(testCase.id))
|
|
32
|
+
: configuredCases;
|
|
33
|
+
const requestedCaseTags = input.requestedCaseTags ?? new Set();
|
|
34
|
+
return requestedCaseTags.size > 0
|
|
35
|
+
? selectedCases.filter((testCase) => hasEveryLiveEvalTag(testCase.metadata?.tags ?? [], requestedCaseTags))
|
|
36
|
+
: selectedCases;
|
|
37
|
+
}
|
|
38
|
+
/** Resolves live eval requested case IDs. */
|
|
39
|
+
export function resolveLiveEvalRequestedCaseIds(input) {
|
|
40
|
+
const resolved = new Set(input.requestedCaseIds);
|
|
41
|
+
const requestedCaseSetId = input.requestedCaseSetId?.trim();
|
|
42
|
+
if (!requestedCaseSetId) {
|
|
43
|
+
return resolved;
|
|
44
|
+
}
|
|
45
|
+
const caseIds = input.caseSets[requestedCaseSetId];
|
|
46
|
+
if (!caseIds) {
|
|
47
|
+
throw new Error(`Unknown AG_UI_EVAL_CASE_SET "${requestedCaseSetId}". Known sets: ${Object.keys(input.caseSets).join(", ")}`);
|
|
48
|
+
}
|
|
49
|
+
for (const caseId of caseIds) {
|
|
50
|
+
resolved.add(caseId);
|
|
51
|
+
}
|
|
52
|
+
return resolved;
|
|
53
|
+
}
|
|
54
|
+
function buildRuntimeCounts(results, runtime) {
|
|
55
|
+
return {
|
|
56
|
+
passed: results.filter((result) => result.runtime === runtime && result.status === "pass").length,
|
|
57
|
+
failed: results.filter((result) => result.runtime === runtime && result.status === "fail").length,
|
|
58
|
+
skipped: results.filter((result) => result.runtime === runtime && result.status === "skip").length,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
/** Builds live eval runtime summary. */
|
|
62
|
+
export function buildLiveEvalRuntimeSummary(runtimes, results) {
|
|
63
|
+
const empty = { passed: 0, failed: 0, skipped: 0 };
|
|
64
|
+
const summary = {
|
|
65
|
+
framework: empty,
|
|
66
|
+
};
|
|
67
|
+
for (const runtime of runtimes) {
|
|
68
|
+
summary[runtime] = buildRuntimeCounts(results, runtime);
|
|
69
|
+
}
|
|
70
|
+
return summary;
|
|
71
|
+
}
|
|
72
|
+
/** Builds live eval status summary. */
|
|
73
|
+
export function buildLiveEvalStatusSummary(results) {
|
|
74
|
+
return {
|
|
75
|
+
passed: results.filter((result) => result.status === "pass").length,
|
|
76
|
+
failed: results.filter((result) => result.status === "fail").length,
|
|
77
|
+
skipped: results.filter((result) => result.status === "skip").length,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
export interface LiveEvalRequestBody {
|
|
2
|
+
threadId: string;
|
|
3
|
+
runId: string;
|
|
4
|
+
state: Record<string, string>;
|
|
5
|
+
tools: unknown[];
|
|
6
|
+
context: unknown[];
|
|
7
|
+
forwardedProps?: {
|
|
8
|
+
veryfront: Record<string, unknown>;
|
|
9
|
+
};
|
|
10
|
+
messages: Array<{
|
|
11
|
+
id: string;
|
|
12
|
+
role: "user";
|
|
13
|
+
content: string;
|
|
14
|
+
}>;
|
|
15
|
+
}
|
|
16
|
+
/** Input payload for build live eval request body. */
|
|
17
|
+
export interface BuildLiveEvalRequestBodyInput {
|
|
18
|
+
testCaseId: string;
|
|
19
|
+
prompt: string;
|
|
20
|
+
metadata?: Record<string, string>;
|
|
21
|
+
projectId: string | null;
|
|
22
|
+
branchId?: string;
|
|
23
|
+
model?: string;
|
|
24
|
+
conversationId?: string | null;
|
|
25
|
+
allowedTools?: string[];
|
|
26
|
+
forceRuntimeOverrides?: boolean;
|
|
27
|
+
maxSteps?: number;
|
|
28
|
+
}
|
|
29
|
+
/** Builds live eval request body. */
|
|
30
|
+
export declare function buildLiveEvalRequestBody(input: BuildLiveEvalRequestBodyInput): LiveEvalRequestBody;
|
|
31
|
+
//# sourceMappingURL=request.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../../../../../src/src/eval/agent-service/live-evals/request.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,KAAK,EAAE,OAAO,EAAE,CAAC;IACjB,OAAO,EAAE,OAAO,EAAE,CAAC;IACnB,cAAc,CAAC,EAAE;QACf,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,CAAC;IACF,QAAQ,EAAE,KAAK,CAAC;QACd,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;CACJ;AAED,sDAAsD;AACtD,MAAM,WAAW,6BAA6B;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,qCAAqC;AACrC,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,6BAA6B,GACnC,mBAAmB,CA6CrB"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/** Public API contract for live eval request body. */
|
|
2
|
+
import * as dntShim from "../../../../_dnt.shims.js";
|
|
3
|
+
/** Builds live eval request body. */
|
|
4
|
+
export function buildLiveEvalRequestBody(input) {
|
|
5
|
+
const veryfront = {};
|
|
6
|
+
if (input.projectId) {
|
|
7
|
+
veryfront.projectId = input.projectId;
|
|
8
|
+
}
|
|
9
|
+
if (input.conversationId) {
|
|
10
|
+
veryfront.conversationId = input.conversationId;
|
|
11
|
+
}
|
|
12
|
+
if (input.branchId) {
|
|
13
|
+
veryfront.branchId = input.branchId;
|
|
14
|
+
}
|
|
15
|
+
if (input.model) {
|
|
16
|
+
veryfront.model = input.model;
|
|
17
|
+
}
|
|
18
|
+
if (input.allowedTools || input.forceRuntimeOverrides) {
|
|
19
|
+
veryfront.runtimeOverrides = {
|
|
20
|
+
allowedTools: input.allowedTools ?? [],
|
|
21
|
+
...(input.maxSteps ? { maxSteps: input.maxSteps } : {}),
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
return {
|
|
25
|
+
threadId: dntShim.crypto.randomUUID(),
|
|
26
|
+
runId: `eval-run-${dntShim.crypto.randomUUID()}`,
|
|
27
|
+
state: {
|
|
28
|
+
evalCase: input.testCaseId,
|
|
29
|
+
...(input.metadata ?? {}),
|
|
30
|
+
},
|
|
31
|
+
tools: [],
|
|
32
|
+
context: [],
|
|
33
|
+
...(Object.keys(veryfront).length > 0
|
|
34
|
+
? {
|
|
35
|
+
forwardedProps: {
|
|
36
|
+
veryfront,
|
|
37
|
+
},
|
|
38
|
+
}
|
|
39
|
+
: {}),
|
|
40
|
+
messages: [
|
|
41
|
+
{
|
|
42
|
+
id: dntShim.crypto.randomUUID(),
|
|
43
|
+
role: "user",
|
|
44
|
+
content: input.prompt,
|
|
45
|
+
},
|
|
46
|
+
],
|
|
47
|
+
};
|
|
48
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import type { LiveEvalRuntime } from "./performance.js";
|
|
2
|
+
/** Record shape for live eval result. */
|
|
3
|
+
export interface LiveEvalResultRecord {
|
|
4
|
+
id: string;
|
|
5
|
+
label: string;
|
|
6
|
+
runtime: LiveEvalRuntime;
|
|
7
|
+
status: "pass" | "fail" | "skip";
|
|
8
|
+
details: string;
|
|
9
|
+
durationMs: number;
|
|
10
|
+
conversationId?: string;
|
|
11
|
+
runId?: string;
|
|
12
|
+
artifactPaths?: string[];
|
|
13
|
+
traceSignature?: string;
|
|
14
|
+
toolStarts?: string[];
|
|
15
|
+
toolArgsPreview?: string;
|
|
16
|
+
textPreview?: string;
|
|
17
|
+
}
|
|
18
|
+
/** Result returned from create skipped eval. */
|
|
19
|
+
export declare function createSkippedEvalResult(input: {
|
|
20
|
+
id: string;
|
|
21
|
+
label: string;
|
|
22
|
+
runtime: LiveEvalRuntime;
|
|
23
|
+
details: string;
|
|
24
|
+
startedAt: number;
|
|
25
|
+
}): LiveEvalResultRecord;
|
|
26
|
+
/** Result returned from create failed eval. */
|
|
27
|
+
export declare function createFailedEvalResult(input: {
|
|
28
|
+
id: string;
|
|
29
|
+
label: string;
|
|
30
|
+
runtime: LiveEvalRuntime;
|
|
31
|
+
details: string;
|
|
32
|
+
startedAt: number;
|
|
33
|
+
conversationId?: string;
|
|
34
|
+
runId?: string;
|
|
35
|
+
artifactPaths?: string[];
|
|
36
|
+
traceSignature?: string;
|
|
37
|
+
toolStarts?: string[];
|
|
38
|
+
toolArgsPreview?: string;
|
|
39
|
+
textPreview?: string;
|
|
40
|
+
}): LiveEvalResultRecord;
|
|
41
|
+
/** Result returned from create passed eval. */
|
|
42
|
+
export declare function createPassedEvalResult(input: {
|
|
43
|
+
id: string;
|
|
44
|
+
label: string;
|
|
45
|
+
runtime: LiveEvalRuntime;
|
|
46
|
+
details: string;
|
|
47
|
+
startedAt: number;
|
|
48
|
+
conversationId?: string;
|
|
49
|
+
runId?: string;
|
|
50
|
+
artifactPaths?: string[];
|
|
51
|
+
traceSignature?: string;
|
|
52
|
+
toolStarts?: string[];
|
|
53
|
+
toolArgsPreview?: string;
|
|
54
|
+
textPreview?: string;
|
|
55
|
+
}): LiveEvalResultRecord;
|
|
56
|
+
//# sourceMappingURL=result.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"result.d.ts","sourceRoot":"","sources":["../../../../../src/src/eval/agent-service/live-evals/result.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAExD,yCAAyC;AACzC,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,eAAe,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AA2CD,gDAAgD;AAChD,wBAAgB,uBAAuB,CAAC,KAAK,EAAE;IAC7C,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,eAAe,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;CACnB,GAAG,oBAAoB,CASvB;AAED,+CAA+C;AAC/C,wBAAgB,sBAAsB,CAAC,KAAK,EAAE;IAC5C,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,eAAe,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GAAG,oBAAoB,CAEvB;AAED,+CAA+C;AAC/C,wBAAgB,sBAAsB,CAAC,KAAK,EAAE;IAC5C,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,eAAe,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GAAG,oBAAoB,CAEvB"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
function createEvalResult(status, input) {
|
|
2
|
+
return {
|
|
3
|
+
id: input.id,
|
|
4
|
+
label: input.label,
|
|
5
|
+
runtime: input.runtime,
|
|
6
|
+
status,
|
|
7
|
+
details: input.details,
|
|
8
|
+
durationMs: Date.now() - input.startedAt,
|
|
9
|
+
...(input.conversationId ? { conversationId: input.conversationId } : {}),
|
|
10
|
+
...(input.runId ? { runId: input.runId } : {}),
|
|
11
|
+
...(input.artifactPaths?.length ? { artifactPaths: input.artifactPaths } : {}),
|
|
12
|
+
...(input.traceSignature ? { traceSignature: input.traceSignature } : {}),
|
|
13
|
+
...(input.toolStarts ? { toolStarts: input.toolStarts } : {}),
|
|
14
|
+
...(input.toolArgsPreview ? { toolArgsPreview: input.toolArgsPreview } : {}),
|
|
15
|
+
...(input.textPreview ? { textPreview: input.textPreview } : {}),
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
/** Result returned from create skipped eval. */
|
|
19
|
+
export function createSkippedEvalResult(input) {
|
|
20
|
+
return {
|
|
21
|
+
id: input.id,
|
|
22
|
+
label: input.label,
|
|
23
|
+
runtime: input.runtime,
|
|
24
|
+
status: "skip",
|
|
25
|
+
details: input.details,
|
|
26
|
+
durationMs: Date.now() - input.startedAt,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
/** Result returned from create failed eval. */
|
|
30
|
+
export function createFailedEvalResult(input) {
|
|
31
|
+
return createEvalResult("fail", input);
|
|
32
|
+
}
|
|
33
|
+
/** Result returned from create passed eval. */
|
|
34
|
+
export function createPassedEvalResult(input) {
|
|
35
|
+
return createEvalResult("pass", input);
|
|
36
|
+
}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import { type AgUiSseProgressSnapshot as EvalProgressSnapshot, type ParsedAgUiSseRun as ParsedRun } from "../../../agent/index.js";
|
|
2
|
+
import { type LiveEvalRuntime } from "./performance.js";
|
|
3
|
+
import { type LiveEvalCaseMetadata } from "./report.js";
|
|
4
|
+
import { type LiveEvalResultRecord } from "./result.js";
|
|
5
|
+
/** Input payload for prepared live eval. */
|
|
6
|
+
export interface PreparedLiveEvalInput {
|
|
7
|
+
prompt?: string;
|
|
8
|
+
metadata?: Record<string, string>;
|
|
9
|
+
verificationContext?: LiveEvalContext;
|
|
10
|
+
cleanup?: () => Promise<void>;
|
|
11
|
+
startSidecar?: () => Promise<(() => Promise<void>) | void>;
|
|
12
|
+
}
|
|
13
|
+
/** Context for live eval. */
|
|
14
|
+
export interface LiveEvalContext {
|
|
15
|
+
apiUrl: string;
|
|
16
|
+
authToken: string;
|
|
17
|
+
projectId: string | null;
|
|
18
|
+
}
|
|
19
|
+
/** Public API contract for live eval case. */
|
|
20
|
+
export interface LiveEvalCase {
|
|
21
|
+
readonly id: string;
|
|
22
|
+
readonly label: string;
|
|
23
|
+
readonly prompt?: string;
|
|
24
|
+
allowedTools?: string[];
|
|
25
|
+
forceRuntimeOverrides?: boolean;
|
|
26
|
+
requireProject?: boolean;
|
|
27
|
+
maxSteps?: number;
|
|
28
|
+
expectedEventSubsequence?: string[];
|
|
29
|
+
metadata?: LiveEvalCaseMetadata;
|
|
30
|
+
prepare?: (context: LiveEvalContext) => Promise<PreparedLiveEvalInput>;
|
|
31
|
+
verify: (run: ParsedRun, prepared: PreparedLiveEvalInput | null) => string | null | Promise<string | null>;
|
|
32
|
+
}
|
|
33
|
+
interface FileCheckInput {
|
|
34
|
+
filePath: string;
|
|
35
|
+
requiredContent?: string[];
|
|
36
|
+
description?: string;
|
|
37
|
+
}
|
|
38
|
+
/** Public API contract for live eval project file. */
|
|
39
|
+
export interface LiveEvalProjectFile {
|
|
40
|
+
path: string;
|
|
41
|
+
content: string;
|
|
42
|
+
}
|
|
43
|
+
/** Input payload for live eval project file reader. */
|
|
44
|
+
export interface LiveEvalProjectFileReaderInput {
|
|
45
|
+
filePath: string;
|
|
46
|
+
requestTimeoutMs: number;
|
|
47
|
+
}
|
|
48
|
+
/** Configuration used by live eval runner. */
|
|
49
|
+
export interface LiveEvalRunnerConfig {
|
|
50
|
+
endpoint: string;
|
|
51
|
+
authToken: string;
|
|
52
|
+
apiUrl: string;
|
|
53
|
+
projectId: string | null;
|
|
54
|
+
branchId: string | null;
|
|
55
|
+
model: string | null;
|
|
56
|
+
requestTimeoutMs: number;
|
|
57
|
+
progressLogIntervalMs: number;
|
|
58
|
+
enableLlmJudge: boolean;
|
|
59
|
+
fetch?: (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
|
|
60
|
+
log?: (message: string) => void;
|
|
61
|
+
readProjectFile?: (input: LiveEvalProjectFileReaderInput) => Promise<LiveEvalProjectFile | null>;
|
|
62
|
+
}
|
|
63
|
+
interface LiveEvalJudgeInput {
|
|
64
|
+
question: string;
|
|
65
|
+
criteria: string;
|
|
66
|
+
}
|
|
67
|
+
interface LiveEvalJudgeRequest extends LiveEvalJudgeInput {
|
|
68
|
+
answer: string;
|
|
69
|
+
}
|
|
70
|
+
interface LiveEvalJudgeResult {
|
|
71
|
+
pass: boolean;
|
|
72
|
+
reason: string;
|
|
73
|
+
}
|
|
74
|
+
declare function collectPreparedArtifactPaths(prepared: PreparedLiveEvalInput | null): string[];
|
|
75
|
+
interface LiveEvalResultContext {
|
|
76
|
+
id: string;
|
|
77
|
+
label: string;
|
|
78
|
+
runtime: LiveEvalRuntime;
|
|
79
|
+
startedAt: number;
|
|
80
|
+
conversationId?: string | null;
|
|
81
|
+
artifactPaths?: string[];
|
|
82
|
+
}
|
|
83
|
+
interface LiveEvalRunArtifactsInput {
|
|
84
|
+
run: ParsedRun;
|
|
85
|
+
runId?: string;
|
|
86
|
+
traceSignature: string;
|
|
87
|
+
}
|
|
88
|
+
interface LiveEvalRunArtifacts {
|
|
89
|
+
runId?: string;
|
|
90
|
+
traceSignature: string;
|
|
91
|
+
toolStarts: string[];
|
|
92
|
+
toolArgsPreview: string;
|
|
93
|
+
textPreview: string;
|
|
94
|
+
}
|
|
95
|
+
declare function createLiveEvalRunArtifacts(input: LiveEvalRunArtifactsInput): LiveEvalRunArtifacts;
|
|
96
|
+
declare function createFailedRunEvalResult(input: {
|
|
97
|
+
details: string;
|
|
98
|
+
context: LiveEvalResultContext;
|
|
99
|
+
runArtifacts: LiveEvalRunArtifacts;
|
|
100
|
+
}): LiveEvalResultRecord;
|
|
101
|
+
declare function createPassedRunEvalResult(input: {
|
|
102
|
+
details: string;
|
|
103
|
+
context: LiveEvalResultContext;
|
|
104
|
+
runArtifacts: LiveEvalRunArtifacts;
|
|
105
|
+
}): LiveEvalResultRecord;
|
|
106
|
+
declare function createStreamingFailureEvalResult(input: {
|
|
107
|
+
details: string;
|
|
108
|
+
context: LiveEvalResultContext;
|
|
109
|
+
progress: EvalProgressSnapshot;
|
|
110
|
+
}): LiveEvalResultRecord;
|
|
111
|
+
declare function extractRunId(run: ParsedRun): string | null;
|
|
112
|
+
/** Check whether finished is present. */
|
|
113
|
+
export declare function hasFinished(run: ParsedRun): boolean;
|
|
114
|
+
/** Contains skill load helper. */
|
|
115
|
+
export declare function containsSkillLoad(run: ParsedRun, skillId: string): boolean;
|
|
116
|
+
/** Count step started events helper. */
|
|
117
|
+
export declare function countStepStartedEvents(run: ParsedRun): number;
|
|
118
|
+
/** Create live eval case support. */
|
|
119
|
+
export declare function createLiveEvalCaseSupport(config: LiveEvalRunnerConfig): {
|
|
120
|
+
runEval: (testCase: LiveEvalCase, runtime: LiveEvalRuntime) => Promise<LiveEvalResultRecord>;
|
|
121
|
+
verifyFileExists: (input: FileCheckInput) => Promise<string | null>;
|
|
122
|
+
withJudge: (structuralVerify: (run: ParsedRun) => string | null, judgeInput: LiveEvalJudgeInput) => (run: ParsedRun) => Promise<string | null>;
|
|
123
|
+
judgeLlm: (input: LiveEvalJudgeRequest) => Promise<LiveEvalJudgeResult>;
|
|
124
|
+
};
|
|
125
|
+
/** White-box helpers used by live eval runner tests. */
|
|
126
|
+
export declare const liveEvalRunnerInternals: {
|
|
127
|
+
collectPreparedArtifactPaths: typeof collectPreparedArtifactPaths;
|
|
128
|
+
createFailedRunEvalResult: typeof createFailedRunEvalResult;
|
|
129
|
+
createLiveEvalRunArtifacts: typeof createLiveEvalRunArtifacts;
|
|
130
|
+
createPassedRunEvalResult: typeof createPassedRunEvalResult;
|
|
131
|
+
createStreamingFailureEvalResult: typeof createStreamingFailureEvalResult;
|
|
132
|
+
extractRunId: typeof extractRunId;
|
|
133
|
+
};
|
|
134
|
+
export {};
|
|
135
|
+
//# sourceMappingURL=runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../../../src/src/eval/agent-service/live-evals/runner.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,uBAAuB,IAAI,oBAAoB,EAIpD,KAAK,gBAAgB,IAAI,SAAS,EACnC,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAExD,OAAO,EAAE,KAAK,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,EAIL,KAAK,oBAAoB,EAC1B,MAAM,aAAa,CAAC;AAErB,4CAA4C;AAC5C,MAAM,WAAW,qBAAqB;IACpC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,mBAAmB,CAAC,EAAE,eAAe,CAAC;IACtC,OAAO,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;CAC5D;AAED,6BAA6B;AAC7B,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED,8CAA8C;AAC9C,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,wBAAwB,CAAC,EAAE,MAAM,EAAE,CAAC;IACpC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;IAChC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,eAAe,KAAK,OAAO,CAAC,qBAAqB,CAAC,CAAC;IACvE,MAAM,EAAE,CACN,GAAG,EAAE,SAAS,EACd,QAAQ,EAAE,qBAAqB,GAAG,IAAI,KACnC,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;CAC7C;AAED,UAAU,cAAc;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,sDAAsD;AACtD,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,uDAAuD;AACvD,MAAM,WAAW,8BAA8B;IAC7C,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,8CAA8C;AAC9C,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,cAAc,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,GAAG,GAAG,GAAG,OAAO,EAAE,IAAI,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;IACjF,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,8BAA8B,KAAK,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;CAClG;AAED,UAAU,kBAAkB;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,oBAAqB,SAAQ,kBAAkB;IACvD,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,UAAU,mBAAmB;IAC3B,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAyJD,iBAAS,4BAA4B,CAAC,QAAQ,EAAE,qBAAqB,GAAG,IAAI,GAAG,MAAM,EAAE,CAYtF;AASD,UAAU,qBAAqB;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,eAAe,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;CAC1B;AAED,UAAU,yBAAyB;IACjC,GAAG,EAAE,SAAS,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,UAAU,oBAAoB;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,iBAAS,0BAA0B,CAAC,KAAK,EAAE,yBAAyB,GAAG,oBAAoB,CAQ1F;AAED,iBAAS,yBAAyB,CAAC,KAAK,EAAE;IACxC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,qBAAqB,CAAC;IAC/B,YAAY,EAAE,oBAAoB,CAAC;CACpC,GAAG,oBAAoB,CAevB;AAED,iBAAS,yBAAyB,CAAC,KAAK,EAAE;IACxC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,qBAAqB,CAAC;IAC/B,YAAY,EAAE,oBAAoB,CAAC;CACpC,GAAG,oBAAoB,CAevB;AAED,iBAAS,gCAAgC,CAAC,KAAK,EAAE;IAC/C,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,qBAAqB,CAAC;IAC/B,QAAQ,EAAE,oBAAoB,CAAC;CAChC,GAAG,oBAAoB,CAcvB;AA+FD,iBAAS,YAAY,CAAC,GAAG,EAAE,SAAS,GAAG,MAAM,GAAG,IAAI,CASnD;AAED,yCAAyC;AACzC,wBAAgB,WAAW,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAEnD;AAED,kCAAkC;AAClC,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAE1E;AAED,wCAAwC;AACxC,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,SAAS,GAAG,MAAM,CAE7D;AAED,qCAAqC;AACrC,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,oBAAoB,GAAG;IACvE,OAAO,EAAE,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe,KAAK,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC7F,gBAAgB,EAAE,CAAC,KAAK,EAAE,cAAc,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IACpE,SAAS,EAAE,CACT,gBAAgB,EAAE,CAAC,GAAG,EAAE,SAAS,KAAK,MAAM,GAAG,IAAI,EACnD,UAAU,EAAE,kBAAkB,KAC3B,CAAC,GAAG,EAAE,SAAS,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAChD,QAAQ,EAAE,CAAC,KAAK,EAAE,oBAAoB,KAAK,OAAO,CAAC,mBAAmB,CAAC,CAAC;CACzE,CAoIA;AAED,wDAAwD;AACxD,eAAO,MAAM,uBAAuB;;;;;;;CAOnC,CAAC"}
|