veryfront 0.1.519 → 0.1.521

package/src/src/agent/service/node-telemetry.ts

@@ -1,12 +1,18 @@
+import { tryResolve } from "../../extensions/contracts.js";
+import {
+  type NodeTelemetryInitializeOptions,
+  type NodeTelemetryInstrumentationConfig,
+  type NodeTelemetryLogger,
+  type NodeTelemetryProcessTarget,
+  type NodeTelemetryProvider,
+  NodeTelemetryProviderName,
+} from "../../extensions/tracing/index.js";
+
 export type NodeHostedAgentServiceTelemetryEnv = Record<string, string | undefined>;
 
 export type NodeAgentServiceTelemetryEnv = NodeHostedAgentServiceTelemetryEnv;
 
-export type NodeHostedAgentServiceInstrumentationConfig = {
-  http: boolean;
-  express: boolean;
-  fs: boolean;
-};
+export type NodeHostedAgentServiceInstrumentationConfig = NodeTelemetryInstrumentationConfig;
 
 export type NodeAgentServiceInstrumentationConfig = NodeHostedAgentServiceInstrumentationConfig;
 
@@ -32,16 +38,11 @@ export type ResolveNodeHostedAgentServiceTelemetryConfigOptions = {
 export type ResolveNodeAgentServiceTelemetryConfigOptions =
   ResolveNodeHostedAgentServiceTelemetryConfigOptions;
 
-export type NodeHostedAgentServiceTelemetryLogger = {
-  info(message: string, metadata?: Record<string, unknown>): void;
-  error(message: string, metadata?: Record<string, unknown>): void;
-};
+export type NodeHostedAgentServiceTelemetryLogger = NodeTelemetryLogger;
 
 export type NodeAgentServiceTelemetryLogger = NodeHostedAgentServiceTelemetryLogger;
 
-export type NodeHostedAgentServiceTelemetryProcessTarget = {
-  on(event: "SIGTERM", listener: () => void | Promise<void>): unknown;
-};
+export type NodeHostedAgentServiceTelemetryProcessTarget = NodeTelemetryProcessTarget;
 
 export type NodeAgentServiceTelemetryProcessTarget = NodeHostedAgentServiceTelemetryProcessTarget;
 
@@ -50,6 +51,7 @@ export type InitializeNodeHostedAgentServiceTelemetryOptions =
   & {
     logger?: NodeHostedAgentServiceTelemetryLogger;
     processTarget?: NodeHostedAgentServiceTelemetryProcessTarget;
+    telemetryProvider?: NodeTelemetryProvider;
   };
 
 export type InitializeNodeAgentServiceTelemetryOptions =
@@ -159,54 +161,29 @@ export async function initializeNodeHostedAgentServiceOpenTelemetry(
   }
 
   try {
-    const { NodeSDK } = await import("@opentelemetry/sdk-node");
-    const { resourceFromAttributes } = await import("@opentelemetry/resources");
-    const { BatchSpanProcessor, ParentBasedSampler, TraceIdRatioBasedSampler } = await import(
-      "@opentelemetry/sdk-trace-base"
-    );
-    const { OTLPTraceExporter } = await import("@opentelemetry/exporter-trace-otlp-http");
-    const { getNodeAutoInstrumentations } = await import(
-      "@opentelemetry/auto-instrumentations-node"
-    );
-
-    const resource = resourceFromAttributes({
-      "service.name": options.serviceName,
-      "service.version": options.serviceVersion,
-      "deployment.environment": options.deploymentEnvironment,
-    });
-    const traceExporter = new OTLPTraceExporter({ headers: options.exporterHeaders });
-
-    const sdk = new NodeSDK({
-      resource,
-      sampler: new ParentBasedSampler({
-        root: new TraceIdRatioBasedSampler(options.samplingRatio),
-      }),
-      spanProcessor: new BatchSpanProcessor(traceExporter, {
-        maxExportBatchSize: 100,
-        scheduledDelayMillis: 500,
-      }),
-      instrumentations: [
-        getNodeAutoInstrumentations({
-          "@opentelemetry/instrumentation-fs": { enabled: options.instrumentation.fs },
-          "@opentelemetry/instrumentation-http": { enabled: options.instrumentation.http },
-          "@opentelemetry/instrumentation-express": { enabled: options.instrumentation.express },
-        }),
-      ],
-    });
-
-    sdk.start();
-
-    logInfo(options.logger, "OpenTelemetry initialized", {
+    const telemetryProvider = options.telemetryProvider ??
+      tryResolve<NodeTelemetryProvider>(NodeTelemetryProviderName);
+    if (!telemetryProvider) {
+      logError(
+        options.logger,
+        "Failed to initialize OpenTelemetry",
+        'Missing extension for contract "NodeTelemetryProvider"',
+      );
+      return false;
+    }
+
+    const initializeOptions: NodeTelemetryInitializeOptions = {
       serviceName: options.serviceName,
+      serviceVersion: options.serviceVersion,
+      deploymentEnvironment: options.deploymentEnvironment,
       samplingRatio: options.samplingRatio,
-    });
-
-    options.processTarget?.on("SIGTERM", async () => {
-      await sdk.shutdown();
-      logInfo(options.logger, "OpenTelemetry shutdown complete");
-    });
+      exporterHeaders: options.exporterHeaders,
+      instrumentation: options.instrumentation,
+      logger: options.logger,
+      processTarget: options.processTarget,
+    };
 
-    return true;
+    return await telemetryProvider.initialize(initializeOptions);
   } catch (error) {
     logError(options.logger, "Failed to initialize OpenTelemetry", error);
     return false;

package/src/src/agent/service/registration.ts

@@ -1,86 +1,161 @@
 import * as dntShim from "../../../_dnt.shims.js";
-import { z } from "zod";
-
-export const agentServiceRegistrationModeSchema = z.enum([
-  "auto",
-  "enabled",
-  "disabled",
-]);
-
-export const agentServiceRegistrationConfigSchema = z.object({
-  VERYFRONT_API_URL: z.string().url(),
-  VERYFRONT_API_TOKEN: z.string().min(1).optional(),
-  VERYFRONT_PROJECT_ID: z.string().min(1).optional(),
-  VERYFRONT_AGENT_SERVICE_URL: z.string().url().optional(),
-  VERYFRONT_AGENT_SERVICE_KEY: z.string().min(1).max(128).optional(),
-  VERYFRONT_AGENT_SERVICE_REGISTRATION: agentServiceRegistrationModeSchema,
-  VERYFRONT_AGENT_SERVICE_HEARTBEAT_INTERVAL_MS: z.number().positive(),
-  VERYFRONT_AGENT_SERVICE_REGION: z.string().min(1).max(128).optional(),
-});
-
-export const resolvedAgentServiceRegistrationInputSchema = z.object({
-  apiUrl: z.string().url(),
-  authToken: z.string().min(1),
-  serviceName: z.string().min(1).max(128),
-  serviceKey: z.string().min(1).max(128),
-  scopeKind: z.enum(["global", "project"]),
-  projectId: z.string().min(1).optional(),
-  agentId: z.string().min(1).max(128).optional(),
-  baseUrl: z.string().url(),
-  invokeUrl: z.string().url(),
-  version: z.string().min(1).max(128).optional(),
-  runtime: z.string().min(1).max(128).optional(),
-  region: z.string().min(1).max(128).optional(),
-  heartbeatIntervalMs: z.number().positive(),
-});
-
-const agentPushRuntimeServiceRestSchema = z.object({
-  id: z.string().uuid(),
-  service_name: z.string(),
-  service_key: z.string(),
-  scope_kind: z.enum(["global", "project"]),
-  scope_key: z.string(),
-  project_id: z.string().nullable(),
-  agent_id: z.string().nullable(),
-  base_url: z.string().url(),
-  invoke_url: z.string().url(),
-  status: z.enum(["active", "disabled"]),
-  capabilities: z.unknown().nullable(),
-  metadata: z.unknown().nullable(),
-  version: z.string().nullable(),
-  runtime: z.string().nullable(),
-  region: z.string().nullable(),
-  last_heartbeat_at: z.string().nullable(),
-  created_at: z.string(),
-  updated_at: z.string(),
-});
-
-const agentPushRuntimeServiceResponseSchema = z.object({
-  service: agentPushRuntimeServiceRestSchema,
-});
-
-const registerAgentPushRuntimeServiceRequestSchema = z.object({
-  service_name: z.string().min(1).max(128),
-  service_key: z.string().min(1).max(128),
-  scope_kind: z.enum(["global", "project"]),
-  project_id: z.string().optional(),
-  agent_id: z.string().optional(),
-  base_url: z.string().url(),
-  invoke_url: z.string().url(),
-  version: z.string().optional(),
-  runtime: z.string().optional(),
-  region: z.string().optional(),
-});
-
-export type AgentServiceRegistrationMode = z.infer<typeof agentServiceRegistrationModeSchema>;
-export type AgentServiceRegistrationConfig = z.infer<typeof agentServiceRegistrationConfigSchema>;
-export type ResolvedAgentServiceRegistrationInput = z.infer<
-  typeof resolvedAgentServiceRegistrationInputSchema
->;
-export type AgentPushRuntimeServiceRest = z.infer<typeof agentPushRuntimeServiceRestSchema>;
-export type RegisterAgentPushRuntimeServiceRequest = z.infer<
-  typeof registerAgentPushRuntimeServiceRequestSchema
->;
+import type { Schema, SchemaValidator } from "../../extensions/schema/index.js";
+import { defineSchema } from "../../schemas/define.js";
+import { lazySchema } from "../../schemas/lazy.js";
+
+export type AgentServiceRegistrationMode = "auto" | "enabled" | "disabled";
+export type AgentServiceRegistrationConfig = {
+  VERYFRONT_API_URL: string;
+  VERYFRONT_API_TOKEN?: string;
+  VERYFRONT_PROJECT_ID?: string;
+  VERYFRONT_AGENT_SERVICE_URL?: string;
+  VERYFRONT_AGENT_SERVICE_KEY?: string;
+  VERYFRONT_AGENT_SERVICE_REGISTRATION: AgentServiceRegistrationMode;
+  VERYFRONT_AGENT_SERVICE_HEARTBEAT_INTERVAL_MS: number;
+  VERYFRONT_AGENT_SERVICE_REGION?: string;
+};
+export type ResolvedAgentServiceRegistrationInput = {
+  apiUrl: string;
+  authToken: string;
+  serviceName: string;
+  serviceKey: string;
+  scopeKind: "global" | "project";
+  projectId?: string;
+  agentId?: string;
+  baseUrl: string;
+  invokeUrl: string;
+  version?: string;
+  runtime?: string;
+  region?: string;
+  heartbeatIntervalMs: number;
+};
+export type AgentPushRuntimeServiceRest = {
+  id: string;
+  service_name: string;
+  service_key: string;
+  scope_kind: "global" | "project";
+  scope_key: string;
+  project_id: string | null;
+  agent_id: string | null;
+  base_url: string;
+  invoke_url: string;
+  status: "active" | "disabled";
+  capabilities?: unknown | null;
+  metadata?: unknown | null;
+  version: string | null;
+  runtime: string | null;
+  region: string | null;
+  last_heartbeat_at: string | null;
+  created_at: string;
+  updated_at: string;
+};
+export type RegisterAgentPushRuntimeServiceRequest = {
+  service_name: string;
+  service_key: string;
+  scope_kind: "global" | "project";
+  project_id?: string;
+  agent_id?: string;
+  base_url: string;
+  invoke_url: string;
+  version?: string;
+  runtime?: string;
+  region?: string;
+};
+
+function agentServiceRegistrationMode(
+  v: SchemaValidator,
+): Schema<AgentServiceRegistrationMode> {
+  return v.enum(["auto", "enabled", "disabled"] as const);
+}
+
+export const agentServiceRegistrationModeSchema = lazySchema(
+  defineSchema<AgentServiceRegistrationMode>(agentServiceRegistrationMode),
+);
+
+export const agentServiceRegistrationConfigSchema = lazySchema(
+  defineSchema<AgentServiceRegistrationConfig>((v) =>
+    v.object({
+      VERYFRONT_API_URL: v.string().url(),
+      VERYFRONT_API_TOKEN: v.string().min(1).optional(),
+      VERYFRONT_PROJECT_ID: v.string().min(1).optional(),
+      VERYFRONT_AGENT_SERVICE_URL: v.string().url().optional(),
+      VERYFRONT_AGENT_SERVICE_KEY: v.string().min(1).max(128).optional(),
+      VERYFRONT_AGENT_SERVICE_REGISTRATION: agentServiceRegistrationMode(v),
+      VERYFRONT_AGENT_SERVICE_HEARTBEAT_INTERVAL_MS: v.number().positive(),
+      VERYFRONT_AGENT_SERVICE_REGION: v.string().min(1).max(128).optional(),
+    })
+  ),
+);
+
+export const resolvedAgentServiceRegistrationInputSchema = lazySchema(
+  defineSchema<ResolvedAgentServiceRegistrationInput>((v) =>
+    v.object({
+      apiUrl: v.string().url(),
+      authToken: v.string().min(1),
+      serviceName: v.string().min(1).max(128),
+      serviceKey: v.string().min(1).max(128),
+      scopeKind: v.enum(["global", "project"] as const),
+      projectId: v.string().min(1).optional(),
+      agentId: v.string().min(1).max(128).optional(),
+      baseUrl: v.string().url(),
+      invokeUrl: v.string().url(),
+      version: v.string().min(1).max(128).optional(),
+      runtime: v.string().min(1).max(128).optional(),
+      region: v.string().min(1).max(128).optional(),
+      heartbeatIntervalMs: v.number().positive(),
+    })
+  ),
+);
+
+function agentPushRuntimeServiceRest(
+  v: SchemaValidator,
+): Schema<AgentPushRuntimeServiceRest> {
+  return v.object({
+    id: v.string().uuid(),
+    service_name: v.string(),
+    service_key: v.string(),
+    scope_kind: v.enum(["global", "project"] as const),
+    scope_key: v.string(),
+    project_id: v.string().nullable(),
+    agent_id: v.string().nullable(),
+    base_url: v.string().url(),
+    invoke_url: v.string().url(),
+    status: v.enum(["active", "disabled"] as const),
+    capabilities: v.unknown().nullable(),
+    metadata: v.unknown().nullable(),
+    version: v.string().nullable(),
+    runtime: v.string().nullable(),
+    region: v.string().nullable(),
+    last_heartbeat_at: v.string().nullable(),
+    created_at: v.string(),
+    updated_at: v.string(),
+  });
+}
+
+const agentPushRuntimeServiceResponseSchema = lazySchema(
+  defineSchema<{ service: AgentPushRuntimeServiceRest }>((v) =>
+    v.object({
+      service: agentPushRuntimeServiceRest(v),
+    })
+  ),
+);
+
+const registerAgentPushRuntimeServiceRequestSchema = lazySchema(
+  defineSchema<RegisterAgentPushRuntimeServiceRequest>((v) =>
+    v.object({
+      service_name: v.string().min(1).max(128),
+      service_key: v.string().min(1).max(128),
+      scope_kind: v.enum(["global", "project"] as const),
+      project_id: v.string().optional(),
+      agent_id: v.string().optional(),
+      base_url: v.string().url(),
+      invoke_url: v.string().url(),
+      version: v.string().optional(),
+      runtime: v.string().optional(),
+      region: v.string().optional(),
+    })
+  ),
+);
 
 export type AgentServiceRegistrationLogger = {
   info?: (message: string, metadata?: Record<string, unknown>) => void;

package/src/src/agent/testing/agent-tester.ts

@@ -0,0 +1,202 @@
+/**************************
+ * Agent Testing Utilities
+ *
+ * Utilities for testing agents in development and CI/CD.
+ *
+ * @module veryfront/agent/testing
+ **************************/
+import * as dntShim from "../../../_dnt.shims.js";
+
+
+import type { Agent, AgentResponse, Message } from "../types.js";
+
+export interface TestCase {
+  /** Test name */
+  name: string;
+
+  /** Input to agent */
+  input: string | Message[];
+
+  /** Expected output pattern (regex or string) */
+  expected?: RegExp | string;
+
+  /** Expected tool calls */
+  expectToolCalls?: string[];
+
+  /** Maximum execution time (ms) */
+  timeout?: number;
+
+  /** Custom validator */
+  validate?: (response: AgentResponse) => boolean | Promise<boolean>;
+}
+
+export interface TestResult {
+  /** Test case name */
+  name: string;
+
+  /** Pass/fail */
+  passed: boolean;
+
+  /** Response from agent */
+  response?: AgentResponse;
+
+  /** Error if test failed */
+  error?: string;
+
+  /** Execution time */
+  executionTime: number;
+
+  /** Tool calls made */
+  toolCalls: string[];
+}
+
+export interface TestSuite {
+  /** Suite name */
+  name: string;
+
+  /** Test results */
+  results: TestResult[];
+
+  /** Overall pass/fail */
+  passed: boolean;
+
+  /** Total execution time */
+  totalTime: number;
+}
+
+export async function testAgent(agent: Agent, testCases: TestCase[]): Promise<TestSuite> {
+  const suiteStartTime = Date.now();
+  const results = await Promise.all(testCases.map((testCase) => runTestCase(agent, testCase)));
+  const passed = results.every((r) => r.passed);
+
+  return {
+    name: agent.id,
+    results,
+    passed,
+    totalTime: Date.now() - suiteStartTime,
+  };
+}
+
+async function runTestCase(agent: Agent, testCase: TestCase): Promise<TestResult> {
+  const startTime = Date.now();
+
+  try {
+    const timeoutMs = testCase.timeout ?? 30000;
+    let timeoutId: ReturnType<typeof dntShim.setTimeout> | undefined;
+    let response: AgentResponse;
+    try {
+      response = await Promise.race<AgentResponse>([
+        agent.generate({ input: testCase.input }),
+        new Promise<never>((_, reject) => {
+          timeoutId = dntShim.setTimeout(() => reject(new Error("Test timeout")), timeoutMs);
+        }),
+      ]);
+    } finally {
+      clearTimeout(timeoutId);
+    }
+
+    const executionTime = Date.now() - startTime;
+    const toolCalls = response.toolCalls.map((tc) => tc.name);
+
+    const { passed, error } = await validateTestCase(testCase, response, toolCalls);
+
+    return {
+      name: testCase.name,
+      passed,
+      response,
+      error,
+      executionTime,
+      toolCalls,
+    };
+  } catch (error) {
+    return {
+      name: testCase.name,
+      passed: false,
+      error: error instanceof Error ? error.message : String(error),
+      executionTime: Date.now() - startTime,
+      toolCalls: [],
+    };
+  }
+}
+
+async function validateTestCase(
+  testCase: TestCase,
+  response: AgentResponse,
+  toolCalls: string[],
+): Promise<{ passed: boolean; error?: string }> {
+  const expected = testCase.expected;
+
+  if (expected instanceof RegExp) {
+    if (!expected.test(response.text)) {
+      return {
+        passed: false,
+        error: `Output "${response.text}" does not match pattern ${expected}`,
+      };
+    }
+  } else if (typeof expected === "string") {
+    if (!response.text.includes(expected)) {
+      return {
+        passed: false,
+        error: `Output does not contain expected text: "${expected}"`,
+      };
+    }
+  }
+
+  const expectedTools = testCase.expectToolCalls;
+  if (expectedTools?.length) {
+    const missingTools = expectedTools.filter((t) => !toolCalls.includes(t));
+    if (missingTools.length) {
+      return {
+        passed: false,
+        error: `Expected tool calls not found: ${missingTools.join(", ")}`,
+      };
+    }
+  }
+
+  if (testCase.validate) {
+    try {
+      const passed = await testCase.validate(response);
+      if (!passed) return { passed: false, error: "Custom validation failed" };
+    } catch (error) {
+      return {
+        passed: false,
+        error: `Custom validation error: ${error instanceof Error ? error.message : String(error)}`,
+      };
+    }
+  }
+
+  return { passed: true };
+}
+
+export function printTestResults(suite: TestSuite): void {
+  console.log(`\n=== Test Suite: ${suite.name} ===\n`);
+
+  const passed = suite.results.filter((r) => r.passed).length;
+  const total = suite.results.length;
+
+  for (const [index, result] of suite.results.entries()) {
+    const icon = result.passed ? "✅" : "❌";
+    console.log(`${icon} ${index + 1}. ${result.name}`);
+
+    if (!result.passed && result.error) console.log(`   Error: ${result.error}`);
+    if (result.toolCalls.length) console.log(`   Tools used: ${result.toolCalls.join(", ")}`);
+
+    console.log(`   Time: ${result.executionTime}ms\n`);
+  }
+
+  console.log(`Results: ${passed}/${total} passed`);
+  console.log(`Total time: ${suite.totalTime}ms`);
+  console.log(`Status: ${suite.passed ? "✅ PASSED" : "❌ FAILED"}\n`);
+}
+
+export function assertContains(response: AgentResponse, text: string): boolean {
+  return response.text.toLowerCase().includes(text.toLowerCase());
+}
+
+export function assertToolCalled(response: AgentResponse, toolName: string): boolean {
+  return response.toolCalls.some((tc) => tc.name === toolName);
+}
+
+export function assertCompleted(response: AgentResponse): boolean {
+  return response.status === "completed";
+}

package/src/src/agent/testing/index.ts

@@ -0,0 +1,44 @@
+/**
+ * Agent Testing Utilities
+ *
+ * @module veryfront/agent/testing
+ */
+import "../../../_dnt.polyfills.js";
+
+
+export {
+  assertCompleted,
+  assertContains,
+  assertToolCalled,
+  printTestResults,
+  testAgent,
+  type TestCase,
+  type TestResult,
+  type TestSuite,
+} from "./agent-tester.js";
+
+export {
+  buildFailureSuffix,
+  buildLiveEvalCaseTagSummary,
+  buildLiveEvalRequestBody,
+  type BuildLiveEvalRequestBodyInput,
+  buildLiveEvalRuntimeSummary,
+  buildLiveEvalStatusSummary,
+  buildProgressLine,
+  buildRuntimePerformanceSummary,
+  containsOrderedSubsequence,
+  createFailedEvalResult,
+  createPassedEvalResult,
+  createPlainTextPdf,
+  createSkippedEvalResult,
+  hasEveryLiveEvalTag,
+  type LiveEvalCaseMetadata,
+  type LiveEvalCaseSelectionInput,
+  type LiveEvalResultForPerformance,
+  type LiveEvalResultForReport,
+  type LiveEvalResultRecord,
+  type LiveEvalRuntime,
+  resolveLiveEvalRequestedCaseIds,
+  type RuntimePerformanceSummary,
+  selectLiveEvalCases,
+} from "./live-evals/index.js";