veryfront 0.1.533 → 0.1.535

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/esm/cli/commands/extension/init-command.d.ts +1 -1
  2. package/esm/cli/commands/extension/init-command.d.ts.map +1 -1
  3. package/esm/cli/commands/extension/init-command.js +14 -4
  4. package/esm/deno.d.ts +1 -0
  5. package/esm/deno.js +5 -4
  6. package/esm/extensions/ext-auth-jwt/src/index.d.ts.map +1 -1
  7. package/esm/extensions/ext-auth-jwt/src/index.js +0 -1
  8. package/esm/extensions/ext-bundler-esbuild/src/index.js +4 -4
  9. package/esm/extensions/ext-content-mdx/src/index.d.ts.map +1 -1
  10. package/esm/extensions/ext-content-mdx/src/index.js +4 -1
  11. package/esm/extensions/ext-css-tailwind/src/index.d.ts.map +1 -1
  12. package/esm/extensions/ext-css-tailwind/src/index.js +3 -1
  13. package/esm/extensions/ext-db-sqlite/src/index.d.ts.map +1 -1
  14. package/esm/extensions/ext-db-sqlite/src/index.js +3 -1
  15. package/esm/extensions/ext-document-kreuzberg/src/index.d.ts.map +1 -1
  16. package/esm/extensions/ext-document-kreuzberg/src/index.js +3 -1
  17. package/esm/extensions/ext-llm-anthropic/src/index.d.ts.map +1 -1
  18. package/esm/extensions/ext-llm-anthropic/src/index.js +5 -1
  19. package/esm/extensions/ext-llm-google/src/index.d.ts.map +1 -1
  20. package/esm/extensions/ext-llm-google/src/index.js +5 -1
  21. package/esm/extensions/ext-llm-openai/src/index.d.ts.map +1 -1
  22. package/esm/extensions/ext-llm-openai/src/index.js +5 -1
  23. package/esm/extensions/ext-observability-opentelemetry/src/index.d.ts.map +1 -1
  24. package/esm/extensions/ext-observability-opentelemetry/src/index.js +3 -2
  25. package/esm/extensions/ext-parser-babel/src/index.d.ts.map +1 -1
  26. package/esm/extensions/ext-parser-babel/src/index.js +4 -1
  27. package/esm/extensions/ext-sandbox-shell-tools/src/index.d.ts.map +1 -1
  28. package/esm/extensions/ext-sandbox-shell-tools/src/index.js +4 -1
  29. package/esm/extensions/ext-schema-zod/src/index.d.ts.map +1 -1
  30. package/esm/extensions/ext-schema-zod/src/index.js +4 -1
  31. package/esm/src/agent/testing/durable-run-canaries/cli-runner.d.ts +18 -0
  32. package/esm/src/agent/testing/durable-run-canaries/cli-runner.d.ts.map +1 -0
  33. package/esm/src/agent/testing/durable-run-canaries/cli-runner.js +65 -0
  34. package/esm/src/agent/testing/durable-run-canaries/environment.d.ts +11 -0
  35. package/esm/src/agent/testing/durable-run-canaries/environment.d.ts.map +1 -0
  36. package/esm/src/agent/testing/durable-run-canaries/environment.js +13 -0
  37. package/esm/src/agent/testing/durable-run-canaries/index.d.ts +3 -0
  38. package/esm/src/agent/testing/durable-run-canaries/index.d.ts.map +1 -1
  39. package/esm/src/agent/testing/durable-run-canaries/index.js +3 -0
  40. package/esm/src/agent/testing/durable-run-canaries/validation.d.ts +7 -0
  41. package/esm/src/agent/testing/durable-run-canaries/validation.d.ts.map +1 -0
  42. package/esm/src/agent/testing/durable-run-canaries/validation.js +66 -0
  43. package/esm/src/agent/testing/index.d.ts +2 -2
  44. package/esm/src/agent/testing/index.d.ts.map +1 -1
  45. package/esm/src/agent/testing/index.js +2 -2
  46. package/esm/src/agent/testing/live-evals/cli-runner.d.ts +36 -0
  47. package/esm/src/agent/testing/live-evals/cli-runner.d.ts.map +1 -0
  48. package/esm/src/agent/testing/live-evals/cli-runner.js +143 -0
  49. package/esm/src/agent/testing/live-evals/environment.d.ts +12 -0
  50. package/esm/src/agent/testing/live-evals/environment.d.ts.map +1 -0
  51. package/esm/src/agent/testing/live-evals/environment.js +18 -0
  52. package/esm/src/agent/testing/live-evals/index.d.ts +3 -0
  53. package/esm/src/agent/testing/live-evals/index.d.ts.map +1 -1
  54. package/esm/src/agent/testing/live-evals/index.js +3 -0
  55. package/esm/src/agent/testing/live-evals/preflight.d.ts +9 -0
  56. package/esm/src/agent/testing/live-evals/preflight.d.ts.map +1 -0
  57. package/esm/src/agent/testing/live-evals/preflight.js +25 -0
  58. package/esm/src/extensions/builtin-extensions.d.ts.map +1 -1
  59. package/esm/src/extensions/builtin-extensions.js +4 -1
  60. package/esm/src/extensions/capabilities.d.ts +1 -1
  61. package/esm/src/extensions/capabilities.d.ts.map +1 -1
  62. package/esm/src/extensions/capabilities.js +1 -4
  63. package/esm/src/extensions/discovery.d.ts +2 -1
  64. package/esm/src/extensions/discovery.d.ts.map +1 -1
  65. package/esm/src/extensions/discovery.js +24 -1
  66. package/esm/src/extensions/index.d.ts +1 -1
  67. package/esm/src/extensions/index.d.ts.map +1 -1
  68. package/esm/src/extensions/loader.d.ts.map +1 -1
  69. package/esm/src/extensions/loader.js +26 -11
  70. package/esm/src/extensions/types.d.ts +8 -0
  71. package/esm/src/extensions/types.d.ts.map +1 -1
  72. package/esm/src/extensions/validation.d.ts.map +1 -1
  73. package/esm/src/extensions/validation.js +32 -8
  74. package/esm/src/utils/version-constant.d.ts +1 -1
  75. package/esm/src/utils/version-constant.js +1 -1
  76. package/package.json +1 -1
  77. package/src/cli/commands/extension/init-command.ts +14 -4
  78. package/src/deno.js +5 -4
  79. package/src/deps/esm.sh/@types/react-dom@19.2.3/client.d.ts +1 -1
  80. package/src/deps/esm.sh/react-dom@19.2.4/client.d.ts +1 -1
  81. package/src/extensions/ext-auth-jwt/src/index.ts +0 -1
  82. package/src/extensions/ext-bundler-esbuild/src/index.ts +4 -4
  83. package/src/extensions/ext-content-mdx/src/index.ts +4 -1
  84. package/src/extensions/ext-css-tailwind/src/index.ts +3 -1
  85. package/src/extensions/ext-db-sqlite/src/index.ts +3 -1
  86. package/src/extensions/ext-document-kreuzberg/src/index.ts +3 -1
  87. package/src/extensions/ext-llm-anthropic/src/index.ts +5 -1
  88. package/src/extensions/ext-llm-google/src/index.ts +5 -1
  89. package/src/extensions/ext-llm-openai/src/index.ts +5 -1
  90. package/src/extensions/ext-observability-opentelemetry/src/index.ts +3 -2
  91. package/src/extensions/ext-parser-babel/src/index.ts +4 -1
  92. package/src/extensions/ext-sandbox-shell-tools/src/index.ts +4 -1
  93. package/src/extensions/ext-schema-zod/src/index.ts +4 -1
  94. package/src/src/agent/testing/durable-run-canaries/cli-runner.ts +117 -0
  95. package/src/src/agent/testing/durable-run-canaries/environment.ts +27 -0
  96. package/src/src/agent/testing/durable-run-canaries/index.ts +18 -0
  97. package/src/src/agent/testing/durable-run-canaries/validation.ts +87 -0
  98. package/src/src/agent/testing/index.ts +14 -0
  99. package/src/src/agent/testing/live-evals/cli-runner.ts +234 -0
  100. package/src/src/agent/testing/live-evals/environment.ts +31 -0
  101. package/src/src/agent/testing/live-evals/index.ts +16 -0
  102. package/src/src/agent/testing/live-evals/preflight.ts +42 -0
  103. package/src/src/extensions/builtin-extensions.ts +4 -1
  104. package/src/src/extensions/capabilities.ts +1 -5
  105. package/src/src/extensions/discovery.ts +27 -2
  106. package/src/src/extensions/index.ts +1 -0
  107. package/src/src/extensions/loader.ts +29 -11
  108. package/src/src/extensions/types.ts +10 -0
  109. package/src/src/extensions/validation.ts +47 -7
  110. package/src/src/utils/version-constant.ts +1 -1
  111. package/src/deps/esm.sh/@types/react@19.2.3/global.d.ts +0 -165
  112. package/src/deps/esm.sh/@types/react@19.2.3/index.d.ts +0 -4301
@@ -18,12 +18,17 @@ export {
18
18
  } from "./agent-tester.js";
19
19
 
20
20
  export {
21
+ assertCompleted as assertDurableRunCanaryCompleted,
22
+ assertNoMalformedCreateFileToolCalls,
23
+ collectAssistantText,
21
24
  createDurableRunCanaryApiClient,
22
25
  createDurableRunCanaryRunner,
26
+ DEFAULT_DURABLE_RUN_CANARY_TIMEOUT_MS,
23
27
  type DurableRunCanaryApiClient,
24
28
  type DurableRunCanaryApiConfig,
25
29
  type DurableRunCanaryCase,
26
30
  type DurableRunCanaryCreateRootRunInput,
31
+ type DurableRunCanaryEnvironment,
27
32
  type DurableRunCanaryMessage,
28
33
  type DurableRunCanaryPreparedCase,
29
34
  type DurableRunCanaryResult,
@@ -32,8 +37,11 @@ export {
32
37
  type DurableRunCanaryRunSummary,
33
38
  type DurableRunCanarySendUserMessageInput,
34
39
  type DurableRunCanaryStartRunInput,
40
+ findAssistantMessage,
35
41
  getDurableRunCanaryMessageSchema,
36
42
  parseDurableRunCanaryRunSummary,
43
+ resolveDurableRunCanaryEnvironment,
44
+ stringifyUnknown,
37
45
  } from "./durable-run-canaries/index.js";
38
46
 
39
47
  export {
@@ -61,9 +69,11 @@ export {
61
69
  createPlainTextPdf,
62
70
  createSkippedEvalResult,
63
71
  DEFAULT_LIVE_EVAL_AREA_TAG_RULES,
72
+ DEFAULT_LIVE_EVAL_ENDPOINT,
64
73
  DEFAULT_LIVE_EVAL_OPTIONAL_JUDGE_CASE_PREFIXES,
65
74
  deleteLiveEvalConversation,
66
75
  deleteLiveEvalProjectFile,
76
+ evaluateRuntimeConfidenceEnv,
67
77
  getLiveEvalProjectFile,
68
78
  hasEveryLiveEvalTag,
69
79
  hasFinished,
@@ -80,6 +90,7 @@ export {
80
90
  type LiveEvalConversationInput,
81
91
  type LiveEvalCreateConversationInput,
82
92
  type LiveEvalCreateReleaseInput,
93
+ type LiveEvalEnvironment,
83
94
  type LiveEvalInputRequestInput,
84
95
  type LiveEvalInputRequestRecord,
85
96
  type LiveEvalInputResponseValues,
@@ -98,7 +109,10 @@ export {
98
109
  type LiveEvalSubmitInputResponseInput,
99
110
  type LiveEvalWaitForOpenInputRequestInput,
100
111
  type PreparedLiveEvalInput,
112
+ printRuntimeConfidencePreflight,
113
+ resolveLiveEvalEnvironment,
101
114
  resolveLiveEvalRequestedCaseIds,
115
+ type RuntimeConfidencePreflightResult,
102
116
  type RuntimePerformanceSummary,
103
117
  selectLiveEvalCases,
104
118
  submitLiveEvalInputResponse,
@@ -0,0 +1,234 @@
1
+ import { mkdir, writeFile } from "node:fs/promises";
2
+ import { dirname, resolve } from "node:path";
3
+ import { cwd as getProcessCwd } from "node:process";
4
+ import { buildRuntimePerformanceSummary, type LiveEvalRuntime } from "./performance.js";
5
+ import {
6
+ buildLiveEvalCaseTagSummary,
7
+ buildLiveEvalRuntimeSummary,
8
+ buildLiveEvalStatusSummary,
9
+ resolveLiveEvalRequestedCaseIds,
10
+ selectLiveEvalCases,
11
+ } from "./report.js";
12
+ import {
13
+ containsSkillLoad,
14
+ countStepStartedEvents,
15
+ createLiveEvalCaseSupport,
16
+ hasFinished,
17
+ type LiveEvalCase,
18
+ type LiveEvalRunnerConfig,
19
+ } from "./runner.js";
20
+ import { getLiveEvalProjectFile, type LiveEvalApiContext } from "./api-client.js";
21
+ import { resolveLiveEvalEnvironment } from "./environment.js";
22
+ import type { LiveEvalResultRecord } from "./result.js";
23
+
24
+ type EnvRecord = Record<string, string | undefined>;
25
+
26
+ export interface LiveEvalCliCaseGroups {
27
+ readOnlyCases: LiveEvalCase[];
28
+ writeCases: LiveEvalCase[];
29
+ experimentalWriteCases: LiveEvalCase[];
30
+ }
31
+
32
+ export interface LiveEvalCliCaseFactoryInput {
33
+ authToken: string;
34
+ endpoint: string;
35
+ projectId: string | null;
36
+ branchId: string | null;
37
+ model: string | null;
38
+ requestTimeoutMs: number;
39
+ enableLlmJudge: boolean;
40
+ hasFinished: typeof hasFinished;
41
+ containsSkillLoad: typeof containsSkillLoad;
42
+ countStepStartedEvents: typeof countStepStartedEvents;
43
+ verifyFileExists: ReturnType<typeof createLiveEvalCaseSupport>["verifyFileExists"];
44
+ withJudge: ReturnType<typeof createLiveEvalCaseSupport>["withJudge"];
45
+ judgeLlm: ReturnType<typeof createLiveEvalCaseSupport>["judgeLlm"];
46
+ }
47
+
48
+ export interface RunLiveEvalCliInput {
49
+ env: EnvRecord;
50
+ caseSets: Record<string, readonly string[]>;
51
+ createCases: (input: LiveEvalCliCaseFactoryInput) => LiveEvalCliCaseGroups;
52
+ runtimes?: readonly LiveEvalRuntime[];
53
+ cwd?: string;
54
+ log?: (message: string) => void;
55
+ error?: (message: string) => void;
56
+ createCaseSupport?: (
57
+ config: LiveEvalRunnerConfig,
58
+ ) => ReturnType<typeof createLiveEvalCaseSupport>;
59
+ }
60
+
61
+ function splitCsvEnv(value: string | undefined): Set<string> {
62
+ return new Set(
63
+ (value ?? "")
64
+ .split(",")
65
+ .map((entry) => entry.trim())
66
+ .filter((entry) => entry.length > 0),
67
+ );
68
+ }
69
+
70
+ function createTimestampedReportPath(input: {
71
+ cwd: string;
72
+ directory: string;
73
+ }): string {
74
+ return resolve(
75
+ input.cwd,
76
+ ".omx/logs",
77
+ input.directory,
78
+ `${new Date().toISOString().replaceAll(":", "-").replaceAll(".", "-")}.json`,
79
+ );
80
+ }
81
+
82
+ export async function runLiveEvalCli(input: RunLiveEvalCliInput): Promise<number> {
83
+ const log = input.log ?? console.log;
84
+ const error = input.error ?? console.error;
85
+ const cwd = input.cwd ?? getProcessCwd();
86
+ const { endpoint, authToken, apiUrl, projectId, branchId, model } = resolveLiveEvalEnvironment(
87
+ input.env,
88
+ );
89
+ const requestedRuntimeSelection = input.runtimes ?? ["framework"];
90
+ const runWriteEvals = input.env.AG_UI_EVAL_WRITE === "1";
91
+ const runExperimentalWriteEvals = input.env.AG_UI_EVAL_EXPERIMENTAL === "1";
92
+ const requestTimeoutMs = Number(input.env.AG_UI_EVAL_TIMEOUT_MS ?? "240000");
93
+ const progressLogIntervalMs = Number(input.env.AG_UI_EVAL_PROGRESS_MS ?? "15000");
94
+ const reportPath = input.env.AG_UI_EVAL_REPORT_PATH ??
95
+ createTimestampedReportPath({ cwd, directory: "ag-ui-live-evals" });
96
+ const requestedCaseIds = splitCsvEnv(input.env.AG_UI_EVAL_CASES);
97
+ const requestedCaseTags = splitCsvEnv(input.env.AG_UI_EVAL_TAGS);
98
+ const requestedCaseSetId = input.env.AG_UI_EVAL_CASE_SET?.trim() || null;
99
+ const enableLlmJudge = input.env.AG_UI_EVAL_LLM_JUDGE === "1";
100
+
101
+ const apiContext: LiveEvalApiContext = {
102
+ apiUrl,
103
+ authToken,
104
+ projectId: projectId ?? null,
105
+ };
106
+ const createCaseSupport = input.createCaseSupport ?? createLiveEvalCaseSupport;
107
+ const { judgeLlm, runEval, verifyFileExists, withJudge } = createCaseSupport({
108
+ endpoint,
109
+ authToken,
110
+ apiUrl,
111
+ projectId: projectId ?? null,
112
+ branchId: branchId ?? null,
113
+ model: model ?? null,
114
+ requestTimeoutMs,
115
+ progressLogIntervalMs,
116
+ enableLlmJudge,
117
+ readProjectFile: (readerInput) => getLiveEvalProjectFile(apiContext, readerInput),
118
+ });
119
+
120
+ const { readOnlyCases, writeCases, experimentalWriteCases } = input.createCases({
121
+ authToken,
122
+ endpoint,
123
+ projectId: projectId ?? null,
124
+ branchId: branchId ?? null,
125
+ model: model ?? null,
126
+ requestTimeoutMs,
127
+ enableLlmJudge,
128
+ hasFinished,
129
+ containsSkillLoad,
130
+ countStepStartedEvents,
131
+ verifyFileExists,
132
+ withJudge,
133
+ judgeLlm,
134
+ });
135
+
136
+ if (authToken.length === 0) {
137
+ error("Missing VERYFRONT_TOKEN");
138
+ return 1;
139
+ }
140
+
141
+ log(`AG-UI live evals -> ${endpoint}`);
142
+ log(`Veryfront API -> ${apiUrl}`);
143
+ log(`Project scope -> ${projectId ?? "none"}`);
144
+ log(`Runtime -> ${requestedRuntimeSelection.join(", ")}`);
145
+ log(`Write evals -> ${runWriteEvals ? "enabled" : "disabled"}`);
146
+ log(`Experimental evals -> ${runExperimentalWriteEvals ? "enabled" : "disabled"}`);
147
+ log(`Case set -> ${requestedCaseSetId ?? "none"}`);
148
+ log(`Case tags -> ${requestedCaseTags.size > 0 ? [...requestedCaseTags].join(", ") : "none"}`);
149
+
150
+ const allCases = [...readOnlyCases, ...writeCases, ...experimentalWriteCases];
151
+ const resolvedRequestedCaseIds = resolveLiveEvalRequestedCaseIds({
152
+ caseSets: input.caseSets,
153
+ requestedCaseIds,
154
+ requestedCaseSetId,
155
+ });
156
+ const cases = selectLiveEvalCases({
157
+ allCases,
158
+ readOnlyCases,
159
+ writeCases,
160
+ experimentalWriteCases,
161
+ requestedCaseIds: resolvedRequestedCaseIds,
162
+ requestedCaseTags,
163
+ runWriteEvals,
164
+ runExperimentalWriteEvals,
165
+ });
166
+ const selectedCaseTagSummary = buildLiveEvalCaseTagSummary(cases);
167
+
168
+ if (cases.length === 0) {
169
+ error("No eval cases selected.");
170
+ return 1;
171
+ }
172
+
173
+ const results: LiveEvalResultRecord[] = [];
174
+
175
+ for (const runtime of requestedRuntimeSelection) {
176
+ log(`\n[runtime] ${runtime}`);
177
+ for (const testCase of cases) {
178
+ log(`\n[run] ${runtime} :: ${testCase.label}`);
179
+ const result = await runEval(testCase, runtime);
180
+ results.push(result);
181
+ log(`[${runtime}] [${result.status}] ${result.details}`);
182
+ }
183
+ }
184
+
185
+ const summary = buildLiveEvalStatusSummary(results);
186
+ const runtimeSummary = buildLiveEvalRuntimeSummary(requestedRuntimeSelection, results);
187
+ const runtimePerformanceSummary = buildRuntimePerformanceSummary(results);
188
+
189
+ log("\nSummary");
190
+ log(`passed: ${summary.passed}`);
191
+ log(`failed: ${summary.failed}`);
192
+ log(`skipped: ${summary.skipped}`);
193
+ for (const runtime of requestedRuntimeSelection) {
194
+ const currentRuntimeSummary = runtimeSummary[runtime];
195
+ log(
196
+ `${runtime}: passed=${currentRuntimeSummary.passed} failed=${currentRuntimeSummary.failed} skipped=${currentRuntimeSummary.skipped}`,
197
+ );
198
+ const performance = runtimePerformanceSummary[runtime];
199
+ log(
200
+ `${runtime}: avg=${performance.avgDurationMs}ms p50=${performance.p50DurationMs}ms p95=${performance.p95DurationMs}ms min=${performance.minDurationMs}ms max=${performance.maxDurationMs}ms`,
201
+ );
202
+ }
203
+
204
+ await mkdir(dirname(reportPath), { recursive: true });
205
+ await writeFile(
206
+ reportPath,
207
+ JSON.stringify(
208
+ {
209
+ generatedAt: new Date().toISOString(),
210
+ endpoint,
211
+ apiUrl,
212
+ projectId: projectId ?? null,
213
+ runtimes: requestedRuntimeSelection,
214
+ writeEvals: runWriteEvals,
215
+ requestedCaseIds: [...resolvedRequestedCaseIds],
216
+ requestedCaseTags: [...requestedCaseTags],
217
+ requestedCaseSetId,
218
+ caseMetadata: Object.fromEntries(
219
+ cases.map((testCase) => [testCase.id, testCase.metadata ?? { tags: [] }]),
220
+ ),
221
+ selectedCaseTagSummary,
222
+ results,
223
+ summary,
224
+ runtimeSummary,
225
+ runtimePerformanceSummary,
226
+ },
227
+ null,
228
+ 2,
229
+ ),
230
+ );
231
+ log(`report: ${reportPath}`);
232
+
233
+ return summary.failed > 0 ? 1 : 0;
234
+ }
@@ -0,0 +1,31 @@
1
+ import { type AgentServiceConfigInput, parseAgentServiceConfig } from "../../service/config.js";
2
+
3
+ export interface LiveEvalEnvironment {
4
+ endpoint: string;
5
+ authToken: string;
6
+ apiUrl: string;
7
+ projectId: string | undefined;
8
+ branchId: string | undefined;
9
+ model: string | undefined;
10
+ }
11
+
12
+ export const DEFAULT_LIVE_EVAL_ENDPOINT = "http://127.0.0.1:3001/api/ag-ui";
13
+
14
+ export function resolveLiveEvalEnvironment(
15
+ env: AgentServiceConfigInput = {},
16
+ ): LiveEvalEnvironment {
17
+ return {
18
+ endpoint: typeof env.AG_UI_EVAL_ENDPOINT === "string"
19
+ ? env.AG_UI_EVAL_ENDPOINT
20
+ : DEFAULT_LIVE_EVAL_ENDPOINT,
21
+ authToken: typeof env.VERYFRONT_TOKEN === "string" ? env.VERYFRONT_TOKEN : "",
22
+ apiUrl: typeof env.VERYFRONT_API_URL === "string"
23
+ ? env.VERYFRONT_API_URL
24
+ : parseAgentServiceConfig(env).VERYFRONT_API_URL,
25
+ projectId: typeof env.AG_UI_EVAL_PROJECT_ID === "string"
26
+ ? env.AG_UI_EVAL_PROJECT_ID
27
+ : undefined,
28
+ branchId: typeof env.AG_UI_EVAL_BRANCH_ID === "string" ? env.AG_UI_EVAL_BRANCH_ID : undefined,
29
+ model: typeof env.AG_UI_EVAL_MODEL === "string" ? env.AG_UI_EVAL_MODEL : undefined,
30
+ };
31
+ }
@@ -1,3 +1,14 @@
1
+ export {
2
+ type LiveEvalCliCaseFactoryInput,
3
+ type LiveEvalCliCaseGroups,
4
+ runLiveEvalCli,
5
+ type RunLiveEvalCliInput,
6
+ } from "./cli-runner.js";
7
+ export {
8
+ DEFAULT_LIVE_EVAL_ENDPOINT,
9
+ type LiveEvalEnvironment,
10
+ resolveLiveEvalEnvironment,
11
+ } from "./environment.js";
1
12
  export {
2
13
  cancelLiveEvalInputRequest,
3
14
  createLiveEvalApiClient,
@@ -30,6 +41,11 @@ export {
30
41
  containsOrderedSubsequence,
31
42
  createPlainTextPdf,
32
43
  } from "./formatting.js";
44
+ export {
45
+ evaluateRuntimeConfidenceEnv,
46
+ printRuntimeConfidencePreflight,
47
+ type RuntimeConfidencePreflightResult,
48
+ } from "./preflight.js";
33
49
  export {
34
50
  buildRuntimePerformanceSummary,
35
51
  type LiveEvalResultForPerformance,
@@ -0,0 +1,42 @@
1
+ import { type AgentServiceConfigInput, parseAgentServiceConfig } from "../../service/config.js";
2
+
3
+ export interface RuntimeConfidencePreflightResult {
4
+ ok: boolean;
5
+ resolvedApiUrl: string;
6
+ messages: string[];
7
+ }
8
+
9
+ export function evaluateRuntimeConfidenceEnv(
10
+ env: AgentServiceConfigInput = {},
11
+ resolvedApiUrl: string = parseAgentServiceConfig(env).VERYFRONT_API_URL,
12
+ ): RuntimeConfidencePreflightResult {
13
+ const messages: string[] = [`Resolved VERYFRONT_API_URL: ${resolvedApiUrl}`];
14
+ let hasBlockers = false;
15
+
16
+ if (typeof env.VERYFRONT_TOKEN !== "string" || env.VERYFRONT_TOKEN.length === 0) {
17
+ hasBlockers = true;
18
+ messages.push("BLOCKER: VERYFRONT_TOKEN is missing");
19
+ }
20
+ if (typeof env.AG_UI_EVAL_PROJECT_ID !== "string" || env.AG_UI_EVAL_PROJECT_ID.length === 0) {
21
+ hasBlockers = true;
22
+ messages.push("BLOCKER: AG_UI_EVAL_PROJECT_ID is missing");
23
+ }
24
+
25
+ if (!hasBlockers) {
26
+ messages.push("Runtime-confidence preflight: PASS");
27
+ return { ok: true, resolvedApiUrl, messages };
28
+ }
29
+
30
+ messages.push("Runtime-confidence preflight: FAIL");
31
+ return { ok: false, resolvedApiUrl, messages };
32
+ }
33
+
34
+ export function printRuntimeConfidencePreflight(
35
+ result: RuntimeConfidencePreflightResult,
36
+ output: Pick<Console, "error" | "log"> = console,
37
+ ): void {
38
+ for (const message of result.messages) {
39
+ const writer = result.ok ? output.log : output.error;
40
+ writer(message);
41
+ }
42
+ }
@@ -84,7 +84,10 @@ function createBuiltinLLMProviderExtension(
84
84
  extension: {
85
85
  name: definition.extensionName,
86
86
  version: "0.1.0",
87
- capabilities: [{ type: "contract", name: `LLMProvider:${provider.id}` }],
87
+ contracts: {
88
+ requires: [LLMProviderRegistryName],
89
+ },
90
+ capabilities: [],
88
91
  setup(ctx) {
89
92
  const registry = ctx.require<LLMProviderRegistry>(LLMProviderRegistryName);
90
93
  didRegister = registerBuiltinLLMProvider(registry, provider);
@@ -11,10 +11,6 @@ import type { Capability, ExtensionLogger } from "./types.js";
11
11
  */
12
12
  export function formatCapabilities(capabilities: Capability[]): string[] {
13
13
  return capabilities.map((cap) => {
14
- if (cap.type === "contract") {
15
- return `contract: ${cap.name as string}`;
16
- }
17
-
18
14
  const { type, ...rest } = cap;
19
15
  const extras = Object.keys(rest);
20
16
  if (extras.length === 0) return type;
@@ -53,7 +49,7 @@ const DENO_PERMISSION_MAP: Record<string, PermissionMapping> = {
53
49
 
54
50
  /**
55
51
  * Map capabilities to Deno CLI permission flags.
56
- * Skips non-system capabilities (e.g., "contract").
52
+ * Skips capabilities without a Deno permission mapping.
57
53
  */
58
54
  export function mapToDenoPermissions(capabilities: Capability[]): string[] {
59
55
  const seen = new Set<string>();
@@ -10,7 +10,7 @@ import * as dntShim from "../../_dnt.shims.js";
10
10
 
11
11
 
12
12
  import { join } from "../../deps/jsr.io/@std/path/1.1.4/mod.js";
13
- import type { Capability, ResolvedExtension } from "./types.js";
13
+ import type { Capability, PackageContractMetadata, ResolvedExtension } from "./types.js";
14
14
 
15
15
  /**
16
16
  * Metadata extracted from a package.json that declares itself
@@ -19,6 +19,7 @@ import type { Capability, ResolvedExtension } from "./types.js";
19
19
  export interface PackageMetadata {
20
20
  isExtension: true;
21
21
  capabilities: Capability[];
22
+ contracts?: PackageContractMetadata;
22
23
  }
23
24
 
24
25
  function isCapability(value: unknown): value is Capability {
@@ -29,6 +30,27 @@ function isCapability(value: unknown): value is Capability {
29
30
  return typeof cap.type === "string" && cap.type.length > 0;
30
31
  }
31
32
 
33
+ function parseStringList(value: unknown): string[] | undefined {
34
+ if (!Array.isArray(value)) return undefined;
35
+ const entries = value.filter((entry): entry is string =>
36
+ typeof entry === "string" && entry.length > 0
37
+ );
38
+ return entries.length > 0 ? entries : undefined;
39
+ }
40
+
41
+ function parseContractMetadata(value: unknown): PackageContractMetadata | undefined {
42
+ if (value === null || typeof value !== "object" || Array.isArray(value)) {
43
+ return undefined;
44
+ }
45
+ const raw = value as Record<string, unknown>;
46
+ const contracts: PackageContractMetadata = {};
47
+ const provides = parseStringList(raw.provides);
48
+ const requires = parseStringList(raw.requires);
49
+ if (provides) contracts.provides = provides;
50
+ if (requires) contracts.requires = requires;
51
+ return provides || requires ? contracts : undefined;
52
+ }
53
+
32
54
  /**
33
55
  * Parse veryfront extension metadata from a package.json-like object.
34
56
  *
@@ -55,8 +77,11 @@ export function parsePackageMetadata(
55
77
  const capabilities: Capability[] = Array.isArray(meta.capabilities)
56
78
  ? meta.capabilities.filter(isCapability)
57
79
  : [];
80
+ const contracts = parseContractMetadata(meta.contracts);
58
81
 
59
- return { isExtension: true, capabilities };
82
+ return contracts
83
+ ? { isExtension: true, capabilities, contracts }
84
+ : { isExtension: true, capabilities };
60
85
  }
61
86
 
62
87
  /**
@@ -29,6 +29,7 @@ export type {
29
29
  Extension,
30
30
  ExtensionConfigEntry,
31
31
  ExtensionContext,
32
+ ExtensionContractMetadata,
32
33
  ExtensionFactory,
33
34
  ExtensionLogger,
34
35
  ExtensionSource,
@@ -84,15 +84,11 @@ export class ExtensionLoader {
84
84
  const ext = resolved.extension;
85
85
  extByName.set(ext.name, resolved);
86
86
 
87
- if (ext.provides) {
88
- for (const contract of Object.keys(ext.provides)) {
89
- providerOf.set(contract, ext.name);
90
- }
87
+ for (const contract of providedContractNames(ext)) {
88
+ providerOf.set(contract, ext.name);
91
89
  }
92
90
 
93
- const contracts = ext.capabilities
94
- .filter((c) => c.type === "contract")
95
- .map((c) => c.name as string);
91
+ const contracts = requiredContractNames(ext);
96
92
  if (contracts.length > 0) {
97
93
  consumesContracts.set(ext.name, contracts);
98
94
  }
@@ -168,8 +164,10 @@ export class ExtensionLoader {
168
164
  register(name, impl);
169
165
  }
170
166
 
167
+ const loadOrder = this.topologicalSort(this.flattenPresets(extensions));
168
+
171
169
  // Check for contract conflicts before loading
172
- const conflicts = detectConflicts(extensions);
170
+ const conflicts = detectConflicts(loadOrder);
173
171
  if (conflicts.length > 0) {
174
172
  const details = conflicts
175
173
  .map((c) => `"${c.contract}" provided by: ${c.providers.map((p) => p.name).join(", ")}`)
@@ -183,9 +181,9 @@ export class ExtensionLoader {
183
181
  // provider later in the iteration order cannot overwrite the winning impl
184
182
  // via register(). Without this, merged inputs (config -> package ->
185
183
  // project -> local-file) silently invert the documented source priority.
186
- const contractWinner = selectContractProviders(extensions);
184
+ const contractWinner = selectContractProviders(loadOrder);
187
185
 
188
- for (const resolved of extensions) {
186
+ for (const resolved of loadOrder) {
189
187
  const ext = resolved.extension;
190
188
 
191
189
  const issues = validateExtension(ext);
@@ -209,7 +207,12 @@ export class ExtensionLoader {
209
207
  const ctx: ExtensionContext = {
210
208
  get: <T>(contract: string) => tryResolve<T>(contract),
211
209
  require: <T>(contract: string) => resolveContract<T>(contract),
212
- provide: <T>(contract: string, impl: T) => register(contract, impl),
210
+ provide: <T>(contract: string, impl: T) => {
211
+ const winner = contractWinner.get(contract);
212
+ if (!winner || winner === resolved) {
213
+ register(contract, impl);
214
+ }
215
+ },
213
216
  config: projectConfig,
214
217
  logger: this.logger,
215
218
  };
@@ -262,3 +265,18 @@ export class ExtensionLoader {
262
265
  if (hadSetupExtensions) reset();
263
266
  }
264
267
  }
268
+
269
+ function providedContractNames(ext: Extension): string[] {
270
+ const names = new Set<string>();
271
+ for (const contract of Object.keys(ext.provides ?? {})) {
272
+ names.add(contract);
273
+ }
274
+ for (const contract of ext.contracts?.provides ?? []) {
275
+ names.add(contract);
276
+ }
277
+ return [...names];
278
+ }
279
+
280
+ function requiredContractNames(ext: Extension): string[] {
281
+ return ext.contracts?.requires ?? [];
282
+ }
@@ -13,6 +13,15 @@ export interface Capability {
13
13
  [key: string]: unknown;
14
14
  }
15
15
 
16
+ export interface ExtensionContractMetadata {
17
+ /** Contracts this extension registers dynamically during setup(). */
18
+ provides?: string[];
19
+ /** Contracts this extension needs before setup() runs. */
20
+ requires?: string[];
21
+ }
22
+
23
+ export type PackageContractMetadata = ExtensionContractMetadata;
24
+
16
25
  export interface ExtensionContext {
17
26
  get<T>(contract: string): T | undefined;
18
27
  require<T>(contract: string): T;
@@ -32,6 +41,7 @@ export interface Extension {
32
41
  name: string;
33
42
  version: string;
34
43
  capabilities: Capability[];
44
+ contracts?: ExtensionContractMetadata;
35
45
  setup?(ctx: ExtensionContext): Promise<void> | void;
36
46
  teardown?(): Promise<void> | void;
37
47
  provides?: Record<string, unknown>;
@@ -37,9 +37,7 @@ export function selectContractProviders(
37
37
  ): Map<string, ResolvedExtension> {
38
38
  const winner = new Map<string, ResolvedExtension>();
39
39
  for (const resolved of extensions) {
40
- const provides = resolved.extension.provides;
41
- if (!provides) continue;
42
- for (const contract of Object.keys(provides)) {
40
+ for (const contract of providedContractNames(resolved.extension)) {
43
41
  const current = winner.get(contract);
44
42
  if (
45
43
  !current ||
@@ -52,6 +50,30 @@ export function selectContractProviders(
52
50
  return winner;
53
51
  }
54
52
 
53
+ function providedContractNames(extension: Extension): string[] {
54
+ return [
55
+ ...Object.keys(extension.provides ?? {}),
56
+ ...(extension.contracts?.provides ?? []),
57
+ ];
58
+ }
59
+
60
+ function validateContractList(
61
+ field: string,
62
+ value: unknown,
63
+ issues: string[],
64
+ ): void {
65
+ if (value === undefined) return;
66
+ if (!Array.isArray(value)) {
67
+ issues.push(`${field} must be an array`);
68
+ return;
69
+ }
70
+ for (let i = 0; i < value.length; i++) {
71
+ if (typeof value[i] !== "string" || value[i].length === 0) {
72
+ issues.push(`${field}[${i}] must be a non-empty string`);
73
+ }
74
+ }
75
+ }
76
+
55
77
  /**
56
78
  * Validate the shape of an extension object.
57
79
  * Returns an array of issue descriptions (empty array = valid).
@@ -93,6 +115,27 @@ export function validateExtension(ext: unknown): string[] {
93
115
  }
94
116
  }
95
117
 
118
+ if (candidate.contracts !== undefined) {
119
+ if (
120
+ typeof candidate.contracts !== "object" ||
121
+ candidate.contracts === null ||
122
+ Array.isArray(candidate.contracts)
123
+ ) {
124
+ issues.push("contracts must be an object");
125
+ } else {
126
+ validateContractList(
127
+ "contracts.provides",
128
+ candidate.contracts.provides,
129
+ issues,
130
+ );
131
+ validateContractList(
132
+ "contracts.requires",
133
+ candidate.contracts.requires,
134
+ issues,
135
+ );
136
+ }
137
+ }
138
+
96
139
  return issues;
97
140
  }
98
141
 
@@ -109,10 +152,7 @@ export function detectConflicts(extensions: ResolvedExtension[]): ConflictInfo[]
109
152
  >();
110
153
 
111
154
  for (const resolved of extensions) {
112
- const provides = resolved.extension.provides;
113
- if (!provides) continue;
114
-
115
- for (const contract of Object.keys(provides)) {
155
+ for (const contract of providedContractNames(resolved.extension)) {
116
156
  let list = contractProviders.get(contract);
117
157
  if (!list) {
118
158
  list = [];
@@ -1,3 +1,3 @@
1
1
  // Keep in sync with deno.json version.
2
2
  // scripts/release.ts updates this constant during releases.
3
- export const VERSION = "0.1.533";
3
+ export const VERSION = "0.1.535";