universal-agent-memory 0.6.2 → 0.7.0

package/dist/benchmarks/benchmark.d.ts

@@ -0,0 +1,328 @@
+/**
+ * Terminal-Bench Adapter for UAM
+ *
+ * Assumptions:
+ * - Target: Compare UAM-enabled agents vs naive agents on terminal-style tasks
+ * - Tasks require knowledge of project structure, past decisions, and patterns
+ * - UAM provides persistent memory across task sessions
+ *
+ * What this handles:
+ * - Benchmark task definitions
+ * - Memory-enabled agent wrapper
+ * - Performance comparison framework
+ * - Results aggregation and reporting
+ *
+ * What this does NOT handle:
+ * - Full Terminal-Bench framework integration (use tb run CLI for that)
+ * - Real Docker environment sandboxing
+ * - Multi-agent coordination (future enhancement)
+ */
+import { z } from 'zod';
+export declare const BenchmarkTaskSchema: z.ZodObject<{
+    id: z.ZodString;
+    name: z.ZodString;
+    description: z.ZodString;
+    instruction: z.ZodString;
+    difficulty: z.ZodEnum<["easy", "medium", "hard"]>;
+    category: z.ZodEnum<["memory", "coordination", "code-quality", "performance", "testing"]>;
+    verify: z.ZodFunction<z.ZodTuple<[], z.ZodUnknown>, z.ZodPromise<z.ZodObject<{
+        success: z.ZodBoolean;
+        details: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+    }, "strip", z.ZodTypeAny, {
+        success: boolean;
+        details?: Record<string, any> | undefined;
+    }, {
+        success: boolean;
+        details?: Record<string, any> | undefined;
+    }>>>;
+    estimatedMinutes: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    name: string;
+    description: string;
+    id: string;
+    category: "memory" | "coordination" | "testing" | "code-quality" | "performance";
+    instruction: string;
+    difficulty: "medium" | "easy" | "hard";
+    verify: (...args: unknown[]) => Promise<{
+        success: boolean;
+        details?: Record<string, any> | undefined;
+    }>;
+    estimatedMinutes?: number | undefined;
+}, {
+    name: string;
+    description: string;
+    id: string;
+    category: "memory" | "coordination" | "testing" | "code-quality" | "performance";
+    instruction: string;
+    difficulty: "medium" | "easy" | "hard";
+    verify: (...args: unknown[]) => Promise<{
+        success: boolean;
+        details?: Record<string, any> | undefined;
+    }>;
+    estimatedMinutes?: number | undefined;
+}>;
+export type BenchmarkTask = z.infer<typeof BenchmarkTaskSchema>;
+export declare const AgentExecutionSchema: z.ZodObject<{
+    taskId: z.ZodString;
+    agent: z.ZodString;
+    startTime: z.ZodNumber;
+    endTime: z.ZodNumber;
+    durationMs: z.ZodNumber;
+    success: z.ZodBoolean;
+    attempts: z.ZodNumber;
+    memoryQueries: z.ZodOptional<z.ZodNumber>;
+    tokensUsed: z.ZodOptional<z.ZodNumber>;
+    errors: z.ZodArray<z.ZodString, "many">;
+}, "strip", z.ZodTypeAny, {
+    success: boolean;
+    errors: string[];
+    taskId: string;
+    agent: string;
+    startTime: number;
+    endTime: number;
+    durationMs: number;
+    attempts: number;
+    memoryQueries?: number | undefined;
+    tokensUsed?: number | undefined;
+}, {
+    success: boolean;
+    errors: string[];
+    taskId: string;
+    agent: string;
+    startTime: number;
+    endTime: number;
+    durationMs: number;
+    attempts: number;
+    memoryQueries?: number | undefined;
+    tokensUsed?: number | undefined;
+}>;
+export type AgentExecution = z.infer<typeof AgentExecutionSchema>;
+export declare const BenchmarkResultSchema: z.ZodObject<{
+    taskId: z.ZodString;
+    taskName: z.ZodString;
+    results: z.ZodArray<z.ZodObject<{
+        taskId: z.ZodString;
+        agent: z.ZodString;
+        startTime: z.ZodNumber;
+        endTime: z.ZodNumber;
+        durationMs: z.ZodNumber;
+        success: z.ZodBoolean;
+        attempts: z.ZodNumber;
+        memoryQueries: z.ZodOptional<z.ZodNumber>;
+        tokensUsed: z.ZodOptional<z.ZodNumber>;
+        errors: z.ZodArray<z.ZodString, "many">;
+    }, "strip", z.ZodTypeAny, {
+        success: boolean;
+        errors: string[];
+        taskId: string;
+        agent: string;
+        startTime: number;
+        endTime: number;
+        durationMs: number;
+        attempts: number;
+        memoryQueries?: number | undefined;
+        tokensUsed?: number | undefined;
+    }, {
+        success: boolean;
+        errors: string[];
+        taskId: string;
+        agent: string;
+        startTime: number;
+        endTime: number;
+        durationMs: number;
+        attempts: number;
+        memoryQueries?: number | undefined;
+        tokensUsed?: number | undefined;
+    }>, "many">;
+    summary: z.ZodObject<{
+        uamSuccessRate: z.ZodNumber;
+        naiveSuccessRate: z.ZodNumber;
+        uamAvgDuration: z.ZodNumber;
+        naiveAvgDuration: z.ZodNumber;
+        improvement: z.ZodObject<{
+            successDelta: z.ZodNumber;
+            speedup: z.ZodNumber;
+            memoryQueries: z.ZodNumber;
+        }, "strip", z.ZodTypeAny, {
+            memoryQueries: number;
+            successDelta: number;
+            speedup: number;
+        }, {
+            memoryQueries: number;
+            successDelta: number;
+            speedup: number;
+        }>;
+    }, "strip", z.ZodTypeAny, {
+        uamSuccessRate: number;
+        naiveSuccessRate: number;
+        uamAvgDuration: number;
+        naiveAvgDuration: number;
+        improvement: {
+            memoryQueries: number;
+            successDelta: number;
+            speedup: number;
+        };
+    }, {
+        uamSuccessRate: number;
+        naiveSuccessRate: number;
+        uamAvgDuration: number;
+        naiveAvgDuration: number;
+        improvement: {
+            memoryQueries: number;
+            successDelta: number;
+            speedup: number;
+        };
+    }>;
+}, "strip", z.ZodTypeAny, {
+    summary: {
+        uamSuccessRate: number;
+        naiveSuccessRate: number;
+        uamAvgDuration: number;
+        naiveAvgDuration: number;
+        improvement: {
+            memoryQueries: number;
+            successDelta: number;
+            speedup: number;
+        };
+    };
+    taskId: string;
+    taskName: string;
+    results: {
+        success: boolean;
+        errors: string[];
+        taskId: string;
+        agent: string;
+        startTime: number;
+        endTime: number;
+        durationMs: number;
+        attempts: number;
+        memoryQueries?: number | undefined;
+        tokensUsed?: number | undefined;
+    }[];
+}, {
+    summary: {
+        uamSuccessRate: number;
+        naiveSuccessRate: number;
+        uamAvgDuration: number;
+        naiveAvgDuration: number;
+        improvement: {
+            memoryQueries: number;
+            successDelta: number;
+            speedup: number;
+        };
+    };
+    taskId: string;
+    taskName: string;
+    results: {
+        success: boolean;
+        errors: string[];
+        taskId: string;
+        agent: string;
+        startTime: number;
+        endTime: number;
+        durationMs: number;
+        attempts: number;
+        memoryQueries?: number | undefined;
+        tokensUsed?: number | undefined;
+    }[];
+}>;
+export type BenchmarkResult = z.infer<typeof BenchmarkResultSchema>;
+export declare const OverallBenchmarkStatsSchema: z.ZodObject<{
+    totalTasks: z.ZodNumber;
+    uamSuccess: z.ZodNumber;
+    naiveSuccess: z.ZodNumber;
+    uamSuccessRate: z.ZodNumber;
+    naiveSuccessRate: z.ZodNumber;
+    uamAvgDuration: z.ZodNumber;
+    naiveAvgDuration: z.ZodNumber;
+    overallSpeedup: z.ZodNumber;
+    byDifficulty: z.ZodRecord<z.ZodString, z.ZodObject<{
+        count: z.ZodNumber;
+        uamSuccess: z.ZodNumber;
+        naiveSuccess: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        count: number;
+        uamSuccess: number;
+        naiveSuccess: number;
+    }, {
+        count: number;
+        uamSuccess: number;
+        naiveSuccess: number;
+    }>>;
+    byCategory: z.ZodRecord<z.ZodString, z.ZodObject<{
+        count: z.ZodNumber;
+        uamSuccess: z.ZodNumber;
+        naiveSuccess: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        count: number;
+        uamSuccess: number;
+        naiveSuccess: number;
+    }, {
+        count: number;
+        uamSuccess: number;
+        naiveSuccess: number;
+    }>>;
+}, "strip", z.ZodTypeAny, {
+    uamSuccessRate: number;
+    naiveSuccessRate: number;
+    uamAvgDuration: number;
+    naiveAvgDuration: number;
+    totalTasks: number;
+    uamSuccess: number;
+    naiveSuccess: number;
+    overallSpeedup: number;
+    byDifficulty: Record<string, {
+        count: number;
+        uamSuccess: number;
+        naiveSuccess: number;
+    }>;
+    byCategory: Record<string, {
+        count: number;
+        uamSuccess: number;
+        naiveSuccess: number;
+    }>;
+}, {
+    uamSuccessRate: number;
+    naiveSuccessRate: number;
+    uamAvgDuration: number;
+    naiveAvgDuration: number;
+    totalTasks: number;
+    uamSuccess: number;
+    naiveSuccess: number;
+    overallSpeedup: number;
+    byDifficulty: Record<string, {
+        count: number;
+        uamSuccess: number;
+        naiveSuccess: number;
+    }>;
+    byCategory: Record<string, {
+        count: number;
+        uamSuccess: number;
+        naiveSuccess: number;
+    }>;
+}>;
+export type OverallBenchmarkStats = z.infer<typeof OverallBenchmarkStatsSchema>;
+export declare const BenchmarkConfigSchema: z.ZodObject<{
+    maxAttempts: z.ZodDefault<z.ZodNumber>;
+    timeoutMs: z.ZodDefault<z.ZodNumber>;
+    agents: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
+    memoryEnabled: z.ZodDefault<z.ZodBoolean>;
+    verbose: z.ZodDefault<z.ZodBoolean>;
+    outputDir: z.ZodDefault<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    agents: string[];
+    maxAttempts: number;
+    timeoutMs: number;
+    memoryEnabled: boolean;
+    verbose: boolean;
+    outputDir: string;
+}, {
+    agents?: string[] | undefined;
+    maxAttempts?: number | undefined;
+    timeoutMs?: number | undefined;
+    memoryEnabled?: boolean | undefined;
+    verbose?: boolean | undefined;
+    outputDir?: string | undefined;
+}>;
+export type BenchmarkConfig = z.infer<typeof BenchmarkConfigSchema>;
+//# sourceMappingURL=benchmark.d.ts.map

package/dist/benchmarks/benchmark.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark.d.ts","sourceRoot":"","sources":["../../src/benchmarks/benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAMxB,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAc9B,CAAC;AAEH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAMhE,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAW/B,CAAC;AAEH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAMlE,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAehC,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AAEpE,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAmBtC,CAAC;AAEH,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC;AAMhF,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;EAOhC,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC"}

package/dist/benchmarks/benchmark.js

@@ -0,0 +1,104 @@
+/**
+ * Terminal-Bench Adapter for UAM
+ *
+ * Assumptions:
+ * - Target: Compare UAM-enabled agents vs naive agents on terminal-style tasks
+ * - Tasks require knowledge of project structure, past decisions, and patterns
+ * - UAM provides persistent memory across task sessions
+ *
+ * What this handles:
+ * - Benchmark task definitions
+ * - Memory-enabled agent wrapper
+ * - Performance comparison framework
+ * - Results aggregation and reporting
+ *
+ * What this does NOT handle:
+ * - Full Terminal-Bench framework integration (use tb run CLI for that)
+ * - Real Docker environment sandboxing
+ * - Multi-agent coordination (future enhancement)
+ */
+import { z } from 'zod';
+// ============================================================================
+// Task Types
+// ============================================================================
+export const BenchmarkTaskSchema = z.object({
+    id: z.string(),
+    name: z.string(),
+    description: z.string(),
+    instruction: z.string(),
+    difficulty: z.enum(['easy', 'medium', 'hard']),
+    category: z.enum(['memory', 'coordination', 'code-quality', 'performance', 'testing']),
+    // Verification: checks if agent solved the task correctly
+    verify: z.function().returns(z.promise(z.object({
+        success: z.boolean(),
+        details: z.record(z.any()).optional(),
+    }))),
+    // Estimated time to complete
+    estimatedMinutes: z.number().optional(),
+});
+// ============================================================================
+// Agent Types
+// ============================================================================
+export const AgentExecutionSchema = z.object({
+    taskId: z.string(),
+    agent: z.string(),
+    startTime: z.number(),
+    endTime: z.number(),
+    durationMs: z.number(),
+    success: z.boolean(),
+    attempts: z.number(),
+    memoryQueries: z.number().optional(),
+    tokensUsed: z.number().optional(),
+    errors: z.array(z.string()),
+});
+// ============================================================================
+// Benchmark Result Types
+// ============================================================================
+export const BenchmarkResultSchema = z.object({
+    taskId: z.string(),
+    taskName: z.string(),
+    results: z.array(AgentExecutionSchema),
+    summary: z.object({
+        uamSuccessRate: z.number(),
+        naiveSuccessRate: z.number(),
+        uamAvgDuration: z.number(), // in seconds
+        naiveAvgDuration: z.number(), // in seconds
+        improvement: z.object({
+            successDelta: z.number(), // percentage points
+            speedup: z.number(), // ratio >1 means UAM is faster
+            memoryQueries: z.number(),
+        }),
+    }),
+});
+export const OverallBenchmarkStatsSchema = z.object({
+    totalTasks: z.number(),
+    uamSuccess: z.number(),
+    naiveSuccess: z.number(),
+    uamSuccessRate: z.number(),
+    naiveSuccessRate: z.number(),
+    uamAvgDuration: z.number(),
+    naiveAvgDuration: z.number(),
+    overallSpeedup: z.number(),
+    byDifficulty: z.record(z.object({
+        count: z.number(),
+        uamSuccess: z.number(),
+        naiveSuccess: z.number(),
+    })),
+    byCategory: z.record(z.object({
+        count: z.number(),
+        uamSuccess: z.number(),
+        naiveSuccess: z.number(),
+    })),
+});
+// ============================================================================
+// Benchmark Configuration
+// ============================================================================
+export const BenchmarkConfigSchema = z.object({
+    maxAttempts: z.number().default(3),
+    timeoutMs: z.number().default(300000), // 5 minutes per task
+    agents: z.array(z.string()).default(['uam-agent', 'naive-agent']),
+    memoryEnabled: z.boolean().default(true),
+    verbose: z.boolean().default(false),
+    outputDir: z.string().default('./benchmarks/results'),
+});
+//# sourceMappingURL=benchmark.js.map

package/dist/benchmarks/benchmark.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../../src/benchmarks/benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,+EAA+E;AAC/E,aAAa;AACb,+EAA+E;AAE/E,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1C,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;IACd,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC9C,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,cAAc,EAAE,cAAc,EAAE,aAAa,EAAE,SAAS,CAAC,CAAC;IACtF,0DAA0D;IAC1D,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;QAC9C,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE;QACpB,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE;KACtC,CAAC,CAAC,CAAC;IACJ,6BAA6B;IAC7B,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACxC,CAAC,CAAC;AAIH,+EAA+E;AAC/E,cAAc;AACd,+EAA+E;AAE/E,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;IACtB,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE;IACpB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;IACpB,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;CAC5B,CAAC,CAAC;AAIH,+EAA+E;AAC/E,yBAAyB;AACzB,+EAA+E;AAE/E,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;IACpB,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,oBAAoB,CAAC;IACtC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC;QAChB,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE;QAC1B,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE;QAC5B,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,aAAa;QACzC,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,aAAa;QAC3C,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,oBAAoB;YAC9C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,+BAA+B;YACpD,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE;SAC1B,CAAC;KACH,CAAC;CACH,CAAC,CAAC;AAIH,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC,MAAM,CAAC;IAClD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;IACtB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;IACtB,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE;IACxB,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE;IAC1B,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE;IAC5B,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE;IAC1B,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE;IAC5B,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE;IAC1B,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAC9B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;QACjB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;QACtB,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE;KACzB,CAAC,CAAC;IACH,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAC5B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;QACjB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;QACtB,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE;KACzB,CAAC,CAAC;CACJ,CAAC,CAAC;AAIH,+EAA+E;AAC/E,0BAA0B;AAC1B,+EAA+E;AAE/E,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAClC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,qBAAqB;IAC5D,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;IACjE,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IACxC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACnC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,sBAAsB,CAAC;CACtD,CAAC,CAAC"}

package/dist/benchmarks/execution-verifier.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Execution Verification System for UAM Benchmarks
+ *
+ * Provides real code execution and verification instead of just pattern matching.
+ * Runs generated code in isolated environments and validates output.
+ */
+export interface TestCase {
+    input: string;
+    expectedOutput: string;
+    description?: string;
+}
+export interface VerificationResult {
+    success: boolean;
+    executionSucceeded: boolean;
+    testsRun: number;
+    testsPassed: number;
+    errors: string[];
+    output: string;
+    executionTimeMs: number;
+}
+export interface TaskVerificationConfig {
+    language: 'typescript' | 'javascript' | 'python' | 'shell';
+    setupCommands?: string[];
+    testCases: TestCase[];
+    expectedPatterns?: string[];
+    timeout?: number;
+    requiresExecution?: boolean;
+}
+/**
+ * Verify generated code by executing it
+ */
+export declare function verifyCodeExecution(code: string, config: TaskVerificationConfig): Promise<VerificationResult>;
+/**
+ * Enhanced task verification configurations for benchmark tasks
+ */
+export declare const TASK_VERIFICATION_CONFIGS: Record<string, TaskVerificationConfig>;
+/**
+ * Verify a benchmark task result
+ */
+export declare function verifyBenchmarkTask(taskId: string, generatedCode: string): Promise<VerificationResult>;
+//# sourceMappingURL=execution-verifier.d.ts.map

package/dist/benchmarks/execution-verifier.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"execution-verifier.d.ts","sourceRoot":"","sources":["../../src/benchmarks/execution-verifier.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAOH,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,YAAY,GAAG,YAAY,GAAG,QAAQ,GAAG,OAAO,CAAC;IAC3D,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AA0ED;;GAEG;AACH,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,sBAAsB,GAC7B,OAAO,CAAC,kBAAkB,CAAC,CAyG7B;AAkGD;;GAEG;AACH,eAAO,MAAM,yBAAyB,EAAE,MAAM,CAAC,MAAM,EAAE,sBAAsB,CA4E5E,CAAC;AAEF;;GAEG;AACH,wBAAsB,mBAAmB,CACvC,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,MAAM,GACpB,OAAO,CAAC,kBAAkB,CAAC,CAiB7B"}