veryfront 0.1.914 → 0.1.916
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/deno.js +1 -1
- package/esm/src/chat/provider-errors.d.ts.map +1 -1
- package/esm/src/chat/provider-errors.js +25 -1
- package/esm/src/eval/agent-service/live-evals/runner.d.ts.map +1 -1
- package/esm/src/eval/agent-service/live-evals/runner.js +3 -1
- package/esm/src/server/handlers/request/project-run-execute.handler.d.ts +9 -0
- package/esm/src/server/handlers/request/project-run-execute.handler.d.ts.map +1 -1
- package/esm/src/server/handlers/request/project-run-execute.handler.js +43 -2
- package/esm/src/utils/version-constant.d.ts +1 -1
- package/esm/src/utils/version-constant.js +1 -1
- package/package.json +1 -1
package/esm/deno.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"provider-errors.d.ts","sourceRoot":"","sources":["../../../src/src/chat/provider-errors.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;
|
|
1
|
+
{"version":3,"file":"provider-errors.d.ts","sourceRoot":"","sources":["../../../src/src/chat/provider-errors.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAqCD,4CAA4C;AAC5C,MAAM,MAAM,mBAAmB,GAAG;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,KAAK,CAAA;CAAE,CAAC;AAE7F,0CAA0C;AAC1C,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,mBAAmB,CAMhE;AAgBD,iCAAiC;AACjC,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,OAAO,GAAG,mBAAmB,GAAG,IAAI,CA+B/E;AAED,yCAAyC;AACzC,wBAAgB,oBAAoB,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAOvE;AAiID,sCAAsC;AACtC,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,mBAAmB,CAEtE"}
|
|
@@ -15,6 +15,11 @@ const AI_PROVIDER_SPEND_LIMIT_ERROR = {
|
|
|
15
15
|
message: "The AI provider spend limit has been reached. Try again later or ask an administrator to raise the AI provider spend limit.",
|
|
16
16
|
status: 402,
|
|
17
17
|
};
|
|
18
|
+
const AI_PROVIDER_BILLING_ERROR = {
|
|
19
|
+
code: "AI_PROVIDER_BILLING_ERROR",
|
|
20
|
+
message: "The configured AI provider account cannot process this request. Try a different model, or ask an administrator to check provider billing.",
|
|
21
|
+
status: 502,
|
|
22
|
+
};
|
|
18
23
|
function isErrorRecord(value) {
|
|
19
24
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
20
25
|
}
|
|
@@ -84,6 +89,18 @@ function isAssistantPrefillUnsupportedMessage(message) {
|
|
|
84
89
|
normalizedMessage.includes("conversation must end with a user message");
|
|
85
90
|
return mentionsAssistantPrefill && rejectsAssistantPrefill;
|
|
86
91
|
}
|
|
92
|
+
function isProviderBillingMessage(normalizedMessage) {
|
|
93
|
+
const mentionsProviderApi = normalizedMessage.includes("anthropic api") ||
|
|
94
|
+
normalizedMessage.includes("openai api") ||
|
|
95
|
+
normalizedMessage.includes("google api") ||
|
|
96
|
+
normalizedMessage.includes("mistral api");
|
|
97
|
+
const mentionsProviderBilling = normalizedMessage.includes("plans & billing") ||
|
|
98
|
+
normalizedMessage.includes("provider billing") ||
|
|
99
|
+
normalizedMessage.includes("provider account");
|
|
100
|
+
const mentionsProviderCredits = normalizedMessage.includes("credit balance is too low") ||
|
|
101
|
+
normalizedMessage.includes("provider credits");
|
|
102
|
+
return mentionsProviderApi && mentionsProviderCredits && mentionsProviderBilling;
|
|
103
|
+
}
|
|
87
104
|
function parseKnownProviderBody(body, seen = new WeakSet()) {
|
|
88
105
|
const problemMatch = parseKnownProblemBody(body);
|
|
89
106
|
if (problemMatch) {
|
|
@@ -128,10 +145,14 @@ function parseKnownProviderBody(body, seen = new WeakSet()) {
|
|
|
128
145
|
};
|
|
129
146
|
}
|
|
130
147
|
if (body.type === "invalid_request_error" && typeof body.message === "string") {
|
|
148
|
+
const normalizedMessage = body.message.toLowerCase();
|
|
149
|
+
if (isProviderBillingMessage(normalizedMessage)) {
|
|
150
|
+
return AI_PROVIDER_BILLING_ERROR;
|
|
151
|
+
}
|
|
131
152
|
if (isAssistantPrefillUnsupportedMessage(body.message)) {
|
|
132
153
|
return MODEL_UNSUPPORTED_ASSISTANT_PREFILL_ERROR;
|
|
133
154
|
}
|
|
134
|
-
if (
|
|
155
|
+
if (normalizedMessage.includes("too long")) {
|
|
135
156
|
return { code: "CONTEXT_LENGTH_EXCEEDED", message: "Conversation is too long" };
|
|
136
157
|
}
|
|
137
158
|
}
|
|
@@ -211,6 +232,9 @@ function parseProviderErrorInner(error, seen) {
|
|
|
211
232
|
if (isAssistantPrefillUnsupportedMessage(message)) {
|
|
212
233
|
return MODEL_UNSUPPORTED_ASSISTANT_PREFILL_ERROR;
|
|
213
234
|
}
|
|
235
|
+
if (isProviderBillingMessage(normalizedMessage)) {
|
|
236
|
+
return AI_PROVIDER_BILLING_ERROR;
|
|
237
|
+
}
|
|
214
238
|
if (isCreditLimitMessage(normalizedMessage)) {
|
|
215
239
|
return { code: "INSUFFICIENT_CREDITS", message: "Insufficient AI credits", status: 402 };
|
|
216
240
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../../../src/src/eval/agent-service/live-evals/runner.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,uBAAuB,IAAI,oBAAoB,EAIpD,KAAK,gBAAgB,IAAI,SAAS,EACnC,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAExD,OAAO,EAAE,KAAK,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,EAIL,KAAK,oBAAoB,EAC1B,MAAM,aAAa,CAAC;AAErB,4CAA4C;AAC5C,MAAM,WAAW,qBAAqB;IACpC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,mBAAmB,CAAC,EAAE,eAAe,CAAC;IACtC,OAAO,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;CAC5D;AAED,6BAA6B;AAC7B,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED,8CAA8C;AAC9C,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,wBAAwB,CAAC,EAAE,MAAM,EAAE,CAAC;IACpC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;IAChC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,eAAe,KAAK,OAAO,CAAC,qBAAqB,CAAC,CAAC;IACvE,MAAM,EAAE,CACN,GAAG,EAAE,SAAS,EACd,QAAQ,EAAE,qBAAqB,GAAG,IAAI,KACnC,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;CAC7C;AAED,UAAU,cAAc;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,sDAAsD;AACtD,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,uDAAuD;AACvD,MAAM,WAAW,8BAA8B;IAC7C,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,8CAA8C;AAC9C,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,cAAc,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,GAAG,GAAG,GAAG,OAAO,EAAE,IAAI,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;IACjF,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,8BAA8B,KAAK,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;CAClG;AAED,UAAU,kBAAkB;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,oBAAqB,SAAQ,kBAAkB;IACvD,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,UAAU,mBAAmB;IAC3B,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../../../src/src/eval/agent-service/live-evals/runner.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,uBAAuB,IAAI,oBAAoB,EAIpD,KAAK,gBAAgB,IAAI,SAAS,EACnC,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAExD,OAAO,EAAE,KAAK,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,EAIL,KAAK,oBAAoB,EAC1B,MAAM,aAAa,CAAC;AAErB,4CAA4C;AAC5C,MAAM,WAAW,qBAAqB;IACpC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,mBAAmB,CAAC,EAAE,eAAe,CAAC;IACtC,OAAO,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;CAC5D;AAED,6BAA6B;AAC7B,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED,8CAA8C;AAC9C,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,wBAAwB,CAAC,EAAE,MAAM,EAAE,CAAC;IACpC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;IAChC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,eAAe,KAAK,OAAO,CAAC,qBAAqB,CAAC,CAAC;IACvE,MAAM,EAAE,CACN,GAAG,EAAE,SAAS,EACd,QAAQ,EAAE,qBAAqB,GAAG,IAAI,KACnC,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;CAC7C;AAED,UAAU,cAAc;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,sDAAsD;AACtD,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,uDAAuD;AACvD,MAAM,WAAW,8BAA8B;IAC7C,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,8CAA8C;AAC9C,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,cAAc,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,GAAG,GAAG,GAAG,OAAO,EAAE,IAAI,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;IACjF,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,8BAA8B,KAAK,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;CAClG;AAED,UAAU,kBAAkB;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,oBAAqB,SAAQ,kBAAkB;IACvD,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,UAAU,mBAAmB;IAC3B,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AA8JD,iBAAS,4BAA4B,CAAC,QAAQ,EAAE,qBAAqB,GAAG,IAAI,GAAG,MAAM,EAAE,CAYtF;AASD,UAAU,qBAAqB;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,eAAe,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;CAC1B;AAED,UAAU,yBAAyB;IACjC,GAAG,EAAE,SAAS,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,UAAU,oBAAoB;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,iBAAS,0BAA0B,CAAC,KAAK,EAAE,yBAAyB,GAAG,oBAAoB,CAQ1F;AAED,iBAAS,yBAAyB,CAAC,KAAK,EAAE;IACxC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,qBAAqB,CAAC;IAC/B,YAAY,EAAE,oBAAoB,CAAC;CACpC,GAAG,oBAAoB,CAevB;AAED,iBAAS,yBAAyB,CAAC,KAAK,EAAE;IACxC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,qBAAqB,CAAC;IAC/B,YAAY,EAAE,oBAAoB,CAAC;CACpC,GAAG,oBAAoB,CAevB;AAED,iBAAS,gCAAgC,CAAC,KAAK,EAAE;IAC/C,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,qBAAqB,CAAC;IAC/B,QAAQ,EAAE,oBAAoB,CAAC;CAChC,GAAG,oBAAoB,CAcvB;AA+FD,iBAAS,YAAY,CAAC,GAAG,EAAE,SAAS,GAAG,MAAM,GAAG,IAAI,CASnD;AAED,yCAAyC;AACzC,wBAAgB,WAAW,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAEnD;AAED,kCAAkC;AAClC,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAE1E;AAED,wCAAwC;AACxC,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,SAAS,GAAG,MAAM,CAE7D;AAED,qCAAqC;AACrC,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,oBAAoB,GAAG;IACvE,OAAO,EAAE,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe,KAAK,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC7F,gBAAgB,EAAE,CAAC,KAAK,EAAE,cAAc,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IACpE,SAAS,EAAE,CACT,gBAAgB,EAAE,CAAC,GAAG,EAAE,SAAS,KAAK,MAAM,GAAG,IAAI,EACnD,UAAU,EAAE,kBAAkB,KAC3B,CAAC,GAAG,EAAE,SAAS,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAChD,QAAQ,EAAE,CAAC,KAAK,EAAE,oBAAoB,KAAK,OAAO,CAAC,mBAAmB,CAAC,CAAC;CACzE,CAoIA;AAED,wDAAwD;AACxD,eAAO,MAAM,uBAAuB;;;;;;;CAOnC,CAAC"}
|
|
@@ -22,7 +22,9 @@ CRITERIA: ${input.criteria}
|
|
|
22
22
|
Respond with exactly one line: PASS or FAIL followed by a brief reason.
|
|
23
23
|
Example: "PASS — correctly explains the pattern with accurate details"
|
|
24
24
|
Example: "FAIL — mentions the wrong file convention"`,
|
|
25
|
-
projectId:
|
|
25
|
+
projectId: config.projectId,
|
|
26
|
+
...(config.branchId ? { branchId: config.branchId } : {}),
|
|
27
|
+
...(config.model ? { model: config.model } : {}),
|
|
26
28
|
allowedTools: [],
|
|
27
29
|
forceRuntimeOverrides: true,
|
|
28
30
|
maxSteps: 2,
|
|
@@ -26,6 +26,14 @@ export interface ProjectRunExecuteResponse {
|
|
|
26
26
|
duration_ms?: number;
|
|
27
27
|
artifacts?: unknown[];
|
|
28
28
|
}
|
|
29
|
+
interface EvalReportUploadInput {
|
|
30
|
+
request: ProjectRunExecuteRequest;
|
|
31
|
+
ctx: HandlerContext;
|
|
32
|
+
req: Request;
|
|
33
|
+
report: EvalReport;
|
|
34
|
+
projectReference: string;
|
|
35
|
+
reportPath: string;
|
|
36
|
+
}
|
|
29
37
|
interface WorkflowRunView {
|
|
30
38
|
status: string;
|
|
31
39
|
output?: unknown;
|
|
@@ -69,6 +77,7 @@ export interface ProjectRunExecuteHandlerDeps {
|
|
|
69
77
|
createWorkflowClient(config?: WorkflowClientConfig): WorkflowClientView | Promise<WorkflowClientView>;
|
|
70
78
|
runEval(definition: EvalDefinition, options: RunEvalOptions): Promise<EvalReport>;
|
|
71
79
|
createEvalAgentAdapter(config: AgentServiceEvalAdapterConfig): EvalAgentAdapter;
|
|
80
|
+
uploadEvalReport(input: EvalReportUploadInput): Promise<string | null>;
|
|
72
81
|
ensureProjectDiscovery(ctx: HandlerContext): Promise<void>;
|
|
73
82
|
executeKnowledgeIngest(input: {
|
|
74
83
|
request: ProjectRunExecuteRequest;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"project-run-execute.handler.d.ts","sourceRoot":"","sources":["../../../../../src/src/server/handlers/request/project-run-execute.handler.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAEjE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAE,KAAK,cAAc,EAAgB,MAAM,4BAA4B,CAAC;AAC/E,OAAO,EAAW,KAAK,cAAc,EAAE,KAAK,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAC3F,OAAO,EAAE,KAAK,cAAc,EAAgB,MAAM,4BAA4B,CAAC;AAE/E,OAAO,EACL,KAAK,6BAA6B,EAEnC,MAAM,gCAAgC,CAAC;AAExC,OAAO,KAAK,EACV,gBAAgB,EAChB,cAAc,EAGd,UAAU,EACV,cAAc,EACf,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EAAE,KAAK,kBAAkB,EAAoB,MAAM,sCAAsC,CAAC;AAEjG,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAGvE,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAmB,aAAa,EAAE,MAAM,aAAa,CAAC;AACnG,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAWlD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,GAAG,UAAU,GAAG,MAAM,CAAC;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC;CACvB;AAED,UAAU,eAAe;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;CACrC;AAED,UAAU,mBAAmB;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAED,UAAU,kBAAkB;IAC1B,QAAQ,CAAC,gBAAgB,CAAC,EAAE,SAAS,GAAG,WAAW,CAAC;IACpD,QAAQ,CAAC,QAAQ,EAAE,OAAO,GAAG,IAAI,CAAC;IAClC,KAAK,CACH,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,OAAO,EACd,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAC3B,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAChC,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC;IACvD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,MAAM,WAAW,4BAA4B;IAC3C,YAAY,CACV,MAAM,EAAE,MAAM,EACd,OAAO,EAAE;QACP,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,cAAc,CAAC;QACxB,MAAM,CAAC,EAAE,eAAe,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,GACA,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAAC;IAClC,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IACzD,gBAAgB,CACd,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE;QACP,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,cAAc,CAAC;QACxB,MAAM,CAAC,EAAE,eAAe,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,GACA,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC,CAAC;IACtC,YAAY,CACV,MAAM,EAAE,MAAM,EACd,OAAO,EAAE;QACP,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,cAAc,CAAC;QACxB,MAAM,CAAC,EAAE,eAAe,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,GACA,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAAC;IAClC,oBAAoB,CAClB,MAAM,CAAC,EAAE,oBAAoB,GAC5B,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;IACpD,OAAO,CAAC,UAAU,EAAE,cAAc,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAClF,sBAAsB,CAAC,MAAM,EAAE,6BAA6B,GAAG,gBAAgB,CAAC;IAChF,sBAAsB,CAAC,GAAG,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3D,sBAAsB,CAAC,KAAK,EAAE;QAC5B,OAAO,EAAE,wBAAwB,CAAC;QAClC,GAAG,EAAE,cAAc,CAAC;QACpB,GAAG,EAAE,OAAO,CAAC;KACd,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;IACvC,wBAAwB,CAAC,KAAK,EAAE;QAC9B,OAAO,EAAE,wBAAwB,CAAC;QAClC,GAAG,EAAE,cAAc,CAAC;QACpB,GAAG,EAAE,OAAO,CAAC;KACd,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;IACvC,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,GAAG,IAAI,MAAM,CAAC;CACf;
|
|
1
|
+
{"version":3,"file":"project-run-execute.handler.d.ts","sourceRoot":"","sources":["../../../../../src/src/server/handlers/request/project-run-execute.handler.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAEjE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAE,KAAK,cAAc,EAAgB,MAAM,4BAA4B,CAAC;AAC/E,OAAO,EAAW,KAAK,cAAc,EAAE,KAAK,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAC3F,OAAO,EAAE,KAAK,cAAc,EAAgB,MAAM,4BAA4B,CAAC;AAE/E,OAAO,EACL,KAAK,6BAA6B,EAEnC,MAAM,gCAAgC,CAAC;AAExC,OAAO,KAAK,EACV,gBAAgB,EAChB,cAAc,EAGd,UAAU,EACV,cAAc,EACf,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EAAE,KAAK,kBAAkB,EAAoB,MAAM,sCAAsC,CAAC;AAEjG,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAGvE,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAmB,aAAa,EAAE,MAAM,aAAa,CAAC;AACnG,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAWlD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,GAAG,UAAU,GAAG,MAAM,CAAC;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC;CACvB;AAED,UAAU,qBAAqB;IAC7B,OAAO,EAAE,wBAAwB,CAAC;IAClC,GAAG,EAAE,cAAc,CAAC;IACpB,GAAG,EAAE,OAAO,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,eAAe;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;CACrC;AAED,UAAU,mBAAmB;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAED,UAAU,kBAAkB;IAC1B,QAAQ,CAAC,gBAAgB,CAAC,EAAE,SAAS,GAAG,WAAW,CAAC;IACpD,QAAQ,CAAC,QAAQ,EAAE,OAAO,GAAG,IAAI,CAAC;IAClC,KAAK,CACH,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,OAAO,EACd,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAC3B,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAChC,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC;IACvD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,MAAM,WAAW,4BAA4B;IAC3C,YAAY,CACV,MAAM,EAAE,MAAM,EACd,OAAO,EAAE;QACP,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,cAAc,CAAC;QACxB,MAAM,CAAC,EAAE,eAAe,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,GACA,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAAC;IAClC,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IACzD,gBAAgB,CACd,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE;QACP,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,cAAc,CAAC;QACxB,MAAM,CAAC,EAAE,eAAe,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,GACA,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC,CAAC;IACtC,YAAY,CACV,MAAM,EAAE,MAAM,EACd,OAAO,EAAE;QACP,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,cAAc,CAAC;QACxB,MAAM,CAAC,EAAE,eAAe,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,GACA,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAAC;IAClC,oBAAoB,CAClB,MAAM,CAAC,EAAE,oBAAoB,GAC5B,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;IACpD,OAAO,CAAC,UAAU,EAAE,cAAc,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAClF,sBAAsB,CAAC,MAAM,EAAE,6BAA6B,GAAG,gBAAgB,CAAC;IAChF,gBAAgB,CAAC,KAAK,EAAE,qBAAqB,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IACvE,sBAAsB,CAAC,GAAG,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3D,sBAAsB,CAAC,KAAK,EAAE;QAC5B,OAAO,EAAE,wBAAwB,CAAC;QAClC,GAAG,EAAE,cAAc,CAAC;QACpB,GAAG,EAAE,OAAO,CAAC;KACd,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;IACvC,wBAAwB,CAAC,KAAK,EAAE;QAC9B,OAAO,EAAE,wBAAwB,CAAC;QAClC,GAAG,EAAE,cAAc,CAAC;QACpB,GAAG,EAAE,OAAO,CAAC;KACd,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;IACvC,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,GAAG,IAAI,MAAM,CAAC;CACf;AAi9BD,qBAAa,wBAAyB,SAAQ,WAAW;IAS3C,OAAO,CAAC,QAAQ,CAAC,IAAI;IARjC,QAAQ,EAAE,eAAe,CAMvB;gBAE2B,IAAI,GAAE,4BAA0C;IAIvE,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;CAsExE"}
|
|
@@ -84,6 +84,22 @@ function parseExecuteRequest(value, pathRunId) {
|
|
|
84
84
|
input: parseRecord(value.input),
|
|
85
85
|
};
|
|
86
86
|
}
|
|
87
|
+
function sanitizePathSegment(value, fallback) {
|
|
88
|
+
const normalized = value
|
|
89
|
+
.replace(/^eval:/, "")
|
|
90
|
+
.replace(/[^A-Za-z0-9._-]+/g, "-")
|
|
91
|
+
.replace(/^-+|-+$/g, "")
|
|
92
|
+
.slice(0, 120);
|
|
93
|
+
return normalized || fallback;
|
|
94
|
+
}
|
|
95
|
+
function buildEvalReportPath(report, request) {
|
|
96
|
+
const evalId = sanitizePathSegment(report.definitionId || request.target, "eval");
|
|
97
|
+
const runId = sanitizePathSegment(request.runId, "run");
|
|
98
|
+
return `evals/reports/${evalId}/${runId}.json`;
|
|
99
|
+
}
|
|
100
|
+
function createEvalReportArtifact(path) {
|
|
101
|
+
return { kind: "eval-report", path, contentType: "application/json" };
|
|
102
|
+
}
|
|
87
103
|
function getRunId(pathname) {
|
|
88
104
|
return EXECUTE_PATH_REGEX.exec(pathname)?.[1] ?? null;
|
|
89
105
|
}
|
|
@@ -421,6 +437,14 @@ function createRuntimeApiClient(req, ctx) {
|
|
|
421
437
|
},
|
|
422
438
|
};
|
|
423
439
|
}
|
|
440
|
+
async function uploadEvalReportToProjectFiles(input) {
|
|
441
|
+
const client = createRuntimeApiClient(input.req, input.ctx);
|
|
442
|
+
const encodedProject = encodeURIComponent(input.projectReference);
|
|
443
|
+
const encodedPath = encodeURIComponent(input.reportPath);
|
|
444
|
+
const reportWithPath = { ...input.report, reportPath: input.reportPath };
|
|
445
|
+
const response = await client.put(`/projects/${encodedProject}/files/${encodedPath}`, { content: `${JSON.stringify(reportWithPath, null, 2)}\n` });
|
|
446
|
+
return response.path ?? input.reportPath;
|
|
447
|
+
}
|
|
424
448
|
function getStringArrayConfig(config, keys) {
|
|
425
449
|
for (const key of keys) {
|
|
426
450
|
const value = config[key];
|
|
@@ -677,11 +701,27 @@ async function executeEvalRun(request, ctx, req, deps) {
|
|
|
677
701
|
runId: request.runId,
|
|
678
702
|
});
|
|
679
703
|
const failed = Math.max(report.summary.failed, countFailedEvalRecords(report));
|
|
704
|
+
const projectReference = ctx.projectSlug ?? request.projectId;
|
|
705
|
+
const requestedReportPath = buildEvalReportPath(report, request);
|
|
706
|
+
let uploadError = null;
|
|
707
|
+
const reportPath = await deps.uploadEvalReport({
|
|
708
|
+
request,
|
|
709
|
+
ctx,
|
|
710
|
+
req,
|
|
711
|
+
report,
|
|
712
|
+
projectReference,
|
|
713
|
+
reportPath: requestedReportPath,
|
|
714
|
+
}).catch((error) => {
|
|
715
|
+
uploadError = `Eval report upload failed: ${errorMessage(error)}`;
|
|
716
|
+
return null;
|
|
717
|
+
});
|
|
718
|
+
const result = reportPath ? { ...report, reportPath } : report;
|
|
680
719
|
return {
|
|
681
720
|
success: failed === 0,
|
|
682
|
-
result
|
|
721
|
+
result,
|
|
722
|
+
...(reportPath ? { artifacts: [createEvalReportArtifact(reportPath)] } : {}),
|
|
683
723
|
...(failed > 0 ? { error: `${failed} eval record${failed === 1 ? "" : "s"} failed` } : {}),
|
|
684
|
-
logs:
|
|
724
|
+
logs: uploadError,
|
|
685
725
|
duration_ms: Math.max(0, deps.now() - startedAt),
|
|
686
726
|
};
|
|
687
727
|
}
|
|
@@ -771,6 +811,7 @@ const defaultDeps = {
|
|
|
771
811
|
createWorkflowClient: createRuntimeWorkflowClient,
|
|
772
812
|
runEval,
|
|
773
813
|
createEvalAgentAdapter: createAgentServiceEvalAdapter,
|
|
814
|
+
uploadEvalReport: uploadEvalReportToProjectFiles,
|
|
774
815
|
ensureProjectDiscovery,
|
|
775
816
|
executeKnowledgeIngest: executeKnowledgeIngestRun,
|
|
776
817
|
executeReleaseAssetBuild: executeReleaseAssetBuildRun,
|