veryfront 0.1.533 → 0.1.534

package/src/src/agent/testing/durable-run-canaries/index.ts

@@ -1,3 +1,8 @@
+export {
+  DEFAULT_DURABLE_RUN_CANARY_TIMEOUT_MS,
+  type DurableRunCanaryEnvironment,
+  resolveDurableRunCanaryEnvironment,
+} from "./environment.js";
 export {
   createDurableRunCanaryApiClient,
   createDurableRunCanaryRunner,
@@ -16,3 +21,11 @@ export {
   getDurableRunCanaryMessageSchema,
   parseDurableRunCanaryRunSummary,
 } from "./runner.js";
+
+export {
+  assertCompleted,
+  assertNoMalformedCreateFileToolCalls,
+  collectAssistantText,
+  findAssistantMessage,
+  stringifyUnknown,
+} from "./validation.js";

package/src/src/agent/testing/durable-run-canaries/validation.ts

@@ -0,0 +1,87 @@
+import type { DurableRunCanaryMessage, DurableRunCanaryRunSummary } from "./runner.js";
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+function getStringProperty(record: Record<string, unknown>, key: string): string | null {
+  const value = record[key];
+  return typeof value === "string" ? value : null;
+}
+
+function getToolCallName(part: Record<string, unknown>): string | null {
+  if (part.type === "tool_call") {
+    return getStringProperty(part, "name");
+  }
+
+  if (part.type === "tool-call") {
+    return getStringProperty(part, "toolName");
+  }
+
+  return null;
+}
+
+function hasCreateFileInput(value: unknown): boolean {
+  if (!isRecord(value)) {
+    return false;
+  }
+
+  return getStringProperty(value, "path") !== null && getStringProperty(value, "content") !== null;
+}
+
+export function assertCompleted(run: DurableRunCanaryRunSummary): void {
+  if (run.status !== "completed") {
+    throw new Error(
+      `Expected completed run, got ${run.status} (${run.terminalErrorCode ?? "no-code"}: ${
+        run.terminalErrorMessage ?? "no message"
+      })`,
+    );
+  }
+}
+
+export function findAssistantMessage(
+  messages: DurableRunCanaryMessage[],
+  messageId: string,
+): DurableRunCanaryMessage {
+  const message = messages.find((candidate) => candidate.id === messageId);
+  if (!message) {
+    throw new Error(`Assistant message ${messageId} was not persisted`);
+  }
+  if (message.role !== "assistant") {
+    throw new Error(`Expected assistant message ${messageId}, got role ${message.role}`);
+  }
+  return message;
+}
+
+export function stringifyUnknown(value: unknown): string {
+  if (typeof value === "string") {
+    return value;
+  }
+
+  try {
+    return JSON.stringify(value);
+  } catch {
+    return String(value);
+  }
+}
+
+export function collectAssistantText(message: DurableRunCanaryMessage): string {
+  return message.parts
+    .filter((part) => part.type === "text" && typeof part.text === "string")
+    .map((part) => part.text)
+    .join("\n");
+}
+
+export function assertNoMalformedCreateFileToolCalls(messages: DurableRunCanaryMessage[]): void {
+  for (const message of messages) {
+    for (const part of message.parts) {
+      if (!isRecord(part) || getToolCallName(part) !== "create_file") {
+        continue;
+      }
+
+      if (!hasCreateFileInput(part.input)) {
+        throw new Error("Expected create_file tool_call input to include a path and content");
+      }
+    }
+  }
+}

package/src/src/agent/testing/index.ts

@@ -18,12 +18,17 @@ export {
 } from "./agent-tester.js";
 
 export {
+  assertCompleted as assertDurableRunCanaryCompleted,
+  assertNoMalformedCreateFileToolCalls,
+  collectAssistantText,
   createDurableRunCanaryApiClient,
   createDurableRunCanaryRunner,
+  DEFAULT_DURABLE_RUN_CANARY_TIMEOUT_MS,
   type DurableRunCanaryApiClient,
   type DurableRunCanaryApiConfig,
   type DurableRunCanaryCase,
   type DurableRunCanaryCreateRootRunInput,
+  type DurableRunCanaryEnvironment,
   type DurableRunCanaryMessage,
   type DurableRunCanaryPreparedCase,
   type DurableRunCanaryResult,
@@ -32,8 +37,11 @@ export {
   type DurableRunCanaryRunSummary,
   type DurableRunCanarySendUserMessageInput,
   type DurableRunCanaryStartRunInput,
+  findAssistantMessage,
   getDurableRunCanaryMessageSchema,
   parseDurableRunCanaryRunSummary,
+  resolveDurableRunCanaryEnvironment,
+  stringifyUnknown,
 } from "./durable-run-canaries/index.js";
 
 export {
@@ -61,9 +69,11 @@ export {
   createPlainTextPdf,
   createSkippedEvalResult,
   DEFAULT_LIVE_EVAL_AREA_TAG_RULES,
+  DEFAULT_LIVE_EVAL_ENDPOINT,
   DEFAULT_LIVE_EVAL_OPTIONAL_JUDGE_CASE_PREFIXES,
   deleteLiveEvalConversation,
   deleteLiveEvalProjectFile,
+  evaluateRuntimeConfidenceEnv,
   getLiveEvalProjectFile,
   hasEveryLiveEvalTag,
   hasFinished,
@@ -80,6 +90,7 @@ export {
   type LiveEvalConversationInput,
   type LiveEvalCreateConversationInput,
   type LiveEvalCreateReleaseInput,
+  type LiveEvalEnvironment,
   type LiveEvalInputRequestInput,
   type LiveEvalInputRequestRecord,
   type LiveEvalInputResponseValues,
@@ -98,7 +109,10 @@ export {
   type LiveEvalSubmitInputResponseInput,
   type LiveEvalWaitForOpenInputRequestInput,
   type PreparedLiveEvalInput,
+  printRuntimeConfidencePreflight,
+  resolveLiveEvalEnvironment,
   resolveLiveEvalRequestedCaseIds,
+  type RuntimeConfidencePreflightResult,
   type RuntimePerformanceSummary,
   selectLiveEvalCases,
   submitLiveEvalInputResponse,

package/src/src/agent/testing/live-evals/environment.ts

@@ -0,0 +1,31 @@
+import { type AgentServiceConfigInput, parseAgentServiceConfig } from "../../service/config.js";
+
+export interface LiveEvalEnvironment {
+  endpoint: string;
+  authToken: string;
+  apiUrl: string;
+  projectId: string | undefined;
+  branchId: string | undefined;
+  model: string | undefined;
+}
+
+export const DEFAULT_LIVE_EVAL_ENDPOINT = "http://127.0.0.1:3001/api/ag-ui";
+
+export function resolveLiveEvalEnvironment(
+  env: AgentServiceConfigInput = {},
+): LiveEvalEnvironment {
+  return {
+    endpoint: typeof env.AG_UI_EVAL_ENDPOINT === "string"
+      ? env.AG_UI_EVAL_ENDPOINT
+      : DEFAULT_LIVE_EVAL_ENDPOINT,
+    authToken: typeof env.VERYFRONT_TOKEN === "string" ? env.VERYFRONT_TOKEN : "",
+    apiUrl: typeof env.VERYFRONT_API_URL === "string"
+      ? env.VERYFRONT_API_URL
+      : parseAgentServiceConfig(env).VERYFRONT_API_URL,
+    projectId: typeof env.AG_UI_EVAL_PROJECT_ID === "string"
+      ? env.AG_UI_EVAL_PROJECT_ID
+      : undefined,
+    branchId: typeof env.AG_UI_EVAL_BRANCH_ID === "string" ? env.AG_UI_EVAL_BRANCH_ID : undefined,
+    model: typeof env.AG_UI_EVAL_MODEL === "string" ? env.AG_UI_EVAL_MODEL : undefined,
+  };
+}

package/src/src/agent/testing/live-evals/index.ts

@@ -1,3 +1,8 @@
+export {
+  DEFAULT_LIVE_EVAL_ENDPOINT,
+  type LiveEvalEnvironment,
+  resolveLiveEvalEnvironment,
+} from "./environment.js";
 export {
   cancelLiveEvalInputRequest,
   createLiveEvalApiClient,
@@ -30,6 +35,11 @@ export {
   containsOrderedSubsequence,
   createPlainTextPdf,
 } from "./formatting.js";
+export {
+  evaluateRuntimeConfidenceEnv,
+  printRuntimeConfidencePreflight,
+  type RuntimeConfidencePreflightResult,
+} from "./preflight.js";
 export {
   buildRuntimePerformanceSummary,
   type LiveEvalResultForPerformance,

package/src/src/agent/testing/live-evals/preflight.ts

@@ -0,0 +1,42 @@
+import { type AgentServiceConfigInput, parseAgentServiceConfig } from "../../service/config.js";
+
+export interface RuntimeConfidencePreflightResult {
+  ok: boolean;
+  resolvedApiUrl: string;
+  messages: string[];
+}
+
+export function evaluateRuntimeConfidenceEnv(
+  env: AgentServiceConfigInput = {},
+  resolvedApiUrl: string = parseAgentServiceConfig(env).VERYFRONT_API_URL,
+): RuntimeConfidencePreflightResult {
+  const messages: string[] = [`Resolved VERYFRONT_API_URL: ${resolvedApiUrl}`];
+  let hasBlockers = false;
+
+  if (typeof env.VERYFRONT_TOKEN !== "string" || env.VERYFRONT_TOKEN.length === 0) {
+    hasBlockers = true;
+    messages.push("BLOCKER: VERYFRONT_TOKEN is missing");
+  }
+  if (typeof env.AG_UI_EVAL_PROJECT_ID !== "string" || env.AG_UI_EVAL_PROJECT_ID.length === 0) {
+    hasBlockers = true;
+    messages.push("BLOCKER: AG_UI_EVAL_PROJECT_ID is missing");
+  }
+
+  if (!hasBlockers) {
+    messages.push("Runtime-confidence preflight: PASS");
+    return { ok: true, resolvedApiUrl, messages };
+  }
+
+  messages.push("Runtime-confidence preflight: FAIL");
+  return { ok: false, resolvedApiUrl, messages };
+}
+
+export function printRuntimeConfidencePreflight(
+  result: RuntimeConfidencePreflightResult,
+  output: Pick<Console, "error" | "log"> = console,
+): void {
+  for (const message of result.messages) {
+    const writer = result.ok ? output.log : output.error;
+    writer(message);
+  }
+}

package/src/src/extensions/builtin-extensions.ts

@@ -84,7 +84,10 @@ function createBuiltinLLMProviderExtension(
     extension: {
       name: definition.extensionName,
       version: "0.1.0",
-      capabilities: [{ type: "contract", name: `LLMProvider:${provider.id}` }],
+      contracts: {
+        requires: [LLMProviderRegistryName],
+      },
+      capabilities: [],
       setup(ctx) {
         const registry = ctx.require<LLMProviderRegistry>(LLMProviderRegistryName);
         didRegister = registerBuiltinLLMProvider(registry, provider);

package/src/src/extensions/capabilities.ts

@@ -11,10 +11,6 @@ import type { Capability, ExtensionLogger } from "./types.js";
  */
 export function formatCapabilities(capabilities: Capability[]): string[] {
   return capabilities.map((cap) => {
-    if (cap.type === "contract") {
-      return `contract: ${cap.name as string}`;
-    }
-
     const { type, ...rest } = cap;
     const extras = Object.keys(rest);
     if (extras.length === 0) return type;
@@ -53,7 +49,7 @@ const DENO_PERMISSION_MAP: Record<string, PermissionMapping> = {
 
 /**
  * Map capabilities to Deno CLI permission flags.
- * Skips non-system capabilities (e.g., "contract").
+ * Skips capabilities without a Deno permission mapping.
  */
 export function mapToDenoPermissions(capabilities: Capability[]): string[] {
   const seen = new Set<string>();

package/src/src/extensions/discovery.ts

@@ -10,7 +10,7 @@ import * as dntShim from "../../_dnt.shims.js";
 
 
 import { join } from "../../deps/jsr.io/@std/path/1.1.4/mod.js";
-import type { Capability, ResolvedExtension } from "./types.js";
+import type { Capability, PackageContractMetadata, ResolvedExtension } from "./types.js";
 
 /**
  * Metadata extracted from a package.json that declares itself
@@ -19,6 +19,7 @@ import type { Capability, ResolvedExtension } from "./types.js";
 export interface PackageMetadata {
   isExtension: true;
   capabilities: Capability[];
+  contracts?: PackageContractMetadata;
 }
 
 function isCapability(value: unknown): value is Capability {
@@ -29,6 +30,27 @@ function isCapability(value: unknown): value is Capability {
   return typeof cap.type === "string" && cap.type.length > 0;
 }
 
+function parseStringList(value: unknown): string[] | undefined {
+  if (!Array.isArray(value)) return undefined;
+  const entries = value.filter((entry): entry is string =>
+    typeof entry === "string" && entry.length > 0
+  );
+  return entries.length > 0 ? entries : undefined;
+}
+
+function parseContractMetadata(value: unknown): PackageContractMetadata | undefined {
+  if (value === null || typeof value !== "object" || Array.isArray(value)) {
+    return undefined;
+  }
+  const raw = value as Record<string, unknown>;
+  const contracts: PackageContractMetadata = {};
+  const provides = parseStringList(raw.provides);
+  const requires = parseStringList(raw.requires);
+  if (provides) contracts.provides = provides;
+  if (requires) contracts.requires = requires;
+  return provides || requires ? contracts : undefined;
+}
+
 /**
  * Parse veryfront extension metadata from a package.json-like object.
  *
@@ -55,8 +77,11 @@ export function parsePackageMetadata(
   const capabilities: Capability[] = Array.isArray(meta.capabilities)
     ? meta.capabilities.filter(isCapability)
     : [];
+  const contracts = parseContractMetadata(meta.contracts);
 
-  return { isExtension: true, capabilities };
+  return contracts
+    ? { isExtension: true, capabilities, contracts }
+    : { isExtension: true, capabilities };
 }
 
 /**

package/src/src/extensions/index.ts

@@ -29,6 +29,7 @@ export type {
   Extension,
   ExtensionConfigEntry,
   ExtensionContext,
+  ExtensionContractMetadata,
   ExtensionFactory,
   ExtensionLogger,
   ExtensionSource,

package/src/src/extensions/loader.ts

@@ -84,15 +84,11 @@ export class ExtensionLoader {
       const ext = resolved.extension;
       extByName.set(ext.name, resolved);
 
-      if (ext.provides) {
-        for (const contract of Object.keys(ext.provides)) {
-          providerOf.set(contract, ext.name);
-        }
+      for (const contract of providedContractNames(ext)) {
+        providerOf.set(contract, ext.name);
       }
 
-      const contracts = ext.capabilities
-        .filter((c) => c.type === "contract")
-        .map((c) => c.name as string);
+      const contracts = requiredContractNames(ext);
       if (contracts.length > 0) {
         consumesContracts.set(ext.name, contracts);
       }
@@ -168,8 +164,10 @@ export class ExtensionLoader {
       register(name, impl);
     }
 
+    const loadOrder = this.topologicalSort(this.flattenPresets(extensions));
+
     // Check for contract conflicts before loading
-    const conflicts = detectConflicts(extensions);
+    const conflicts = detectConflicts(loadOrder);
     if (conflicts.length > 0) {
       const details = conflicts
         .map((c) => `"${c.contract}" provided by: ${c.providers.map((p) => p.name).join(", ")}`)
@@ -183,9 +181,9 @@ export class ExtensionLoader {
     // provider later in the iteration order cannot overwrite the winning impl
     // via register(). Without this, merged inputs (config -> package ->
     // project -> local-file) silently invert the documented source priority.
-    const contractWinner = selectContractProviders(extensions);
+    const contractWinner = selectContractProviders(loadOrder);
 
-    for (const resolved of extensions) {
+    for (const resolved of loadOrder) {
       const ext = resolved.extension;
 
       const issues = validateExtension(ext);
@@ -209,7 +207,12 @@ export class ExtensionLoader {
         const ctx: ExtensionContext = {
           get: <T>(contract: string) => tryResolve<T>(contract),
           require: <T>(contract: string) => resolveContract<T>(contract),
-          provide: <T>(contract: string, impl: T) => register(contract, impl),
+          provide: <T>(contract: string, impl: T) => {
+            const winner = contractWinner.get(contract);
+            if (!winner || winner === resolved) {
+              register(contract, impl);
+            }
+          },
           config: projectConfig,
           logger: this.logger,
         };
@@ -262,3 +265,18 @@ export class ExtensionLoader {
     if (hadSetupExtensions) reset();
   }
 }
+
+function providedContractNames(ext: Extension): string[] {
+  const names = new Set<string>();
+  for (const contract of Object.keys(ext.provides ?? {})) {
+    names.add(contract);
+  }
+  for (const contract of ext.contracts?.provides ?? []) {
+    names.add(contract);
+  }
+  return [...names];
+}
+
+function requiredContractNames(ext: Extension): string[] {
+  return ext.contracts?.requires ?? [];
+}

package/src/src/extensions/types.ts

@@ -13,6 +13,15 @@ export interface Capability {
   [key: string]: unknown;
 }
 
+export interface ExtensionContractMetadata {
+  /** Contracts this extension registers dynamically during setup(). */
+  provides?: string[];
+  /** Contracts this extension needs before setup() runs. */
+  requires?: string[];
+}
+
+export type PackageContractMetadata = ExtensionContractMetadata;
+
 export interface ExtensionContext {
   get<T>(contract: string): T | undefined;
   require<T>(contract: string): T;
@@ -32,6 +41,7 @@ export interface Extension {
   name: string;
   version: string;
   capabilities: Capability[];
+  contracts?: ExtensionContractMetadata;
   setup?(ctx: ExtensionContext): Promise<void> | void;
   teardown?(): Promise<void> | void;
   provides?: Record<string, unknown>;

package/src/src/extensions/validation.ts

@@ -37,9 +37,7 @@ export function selectContractProviders(
 ): Map<string, ResolvedExtension> {
   const winner = new Map<string, ResolvedExtension>();
   for (const resolved of extensions) {
-    const provides = resolved.extension.provides;
-    if (!provides) continue;
-    for (const contract of Object.keys(provides)) {
+    for (const contract of providedContractNames(resolved.extension)) {
       const current = winner.get(contract);
       if (
         !current ||
@@ -52,6 +50,30 @@ export function selectContractProviders(
   return winner;
 }
 
+function providedContractNames(extension: Extension): string[] {
+  return [
+    ...Object.keys(extension.provides ?? {}),
+    ...(extension.contracts?.provides ?? []),
+  ];
+}
+
+function validateContractList(
+  field: string,
+  value: unknown,
+  issues: string[],
+): void {
+  if (value === undefined) return;
+  if (!Array.isArray(value)) {
+    issues.push(`${field} must be an array`);
+    return;
+  }
+  for (let i = 0; i < value.length; i++) {
+    if (typeof value[i] !== "string" || value[i].length === 0) {
+      issues.push(`${field}[${i}] must be a non-empty string`);
+    }
+  }
+}
+
 /**
  * Validate the shape of an extension object.
  * Returns an array of issue descriptions (empty array = valid).
@@ -93,6 +115,27 @@ export function validateExtension(ext: unknown): string[] {
     }
   }
 
+  if (candidate.contracts !== undefined) {
+    if (
+      typeof candidate.contracts !== "object" ||
+      candidate.contracts === null ||
+      Array.isArray(candidate.contracts)
+    ) {
+      issues.push("contracts must be an object");
+    } else {
+      validateContractList(
+        "contracts.provides",
+        candidate.contracts.provides,
+        issues,
+      );
+      validateContractList(
+        "contracts.requires",
+        candidate.contracts.requires,
+        issues,
+      );
+    }
+  }
+
   return issues;
 }
 
@@ -109,10 +152,7 @@ export function detectConflicts(extensions: ResolvedExtension[]): ConflictInfo[]
   >();
 
   for (const resolved of extensions) {
-    const provides = resolved.extension.provides;
-    if (!provides) continue;
-
-    for (const contract of Object.keys(provides)) {
+    for (const contract of providedContractNames(resolved.extension)) {
       let list = contractProviders.get(contract);
       if (!list) {
         list = [];

package/src/src/utils/version-constant.ts

@@ -1,3 +1,3 @@
 // Keep in sync with deno.json version.
 // scripts/release.ts updates this constant during releases.
-export const VERSION = "0.1.533";
+export const VERSION = "0.1.534";